网络接收缓存的设计

网络程序，为了提高程序的性能，应尽量减少内存的拷贝次数。


以windows IOCP为例，complete线程在接收到数据后，应该马上将接收到的数据拷贝到
解包缓存，然后立即发起一次新的WSARecv操作。然后再对解包缓存执行操作，解析出逻辑
包。通常的方法是将数据拷贝到一个环形缓冲中，以减少数据的拷贝次数。但在解出一个数据
包之后，免不了还要将一个完整的数据包拷贝到另一个包缓存中，再将这个包提交给应用层处理.


本文介绍一种，拼包方式，以避免从解包缓存copy数据到逻辑包中.

首先介绍逻辑包的组织结构:

//RPacket数据可跨越多个buffer
class RPacket
{
    friend class Connection;
    friend class WPacket;
public:
    RPacket(const RPacket &other):m_buf(other.m_buf),m_head(other.m_head),m_len(other.m_len)
        ,m_readPos(other.m_readPos),m_readBuf(0),m_binBuffer(0),m_binBufferPos(0)
    {
    }    
    
    
    ~RPacket()
    {
        if(m_binBuffer)
            delete[] m_binBuffer;
    }

    bool _nil()
    {
        return m_buf._nil();
    }

    RPacket& operator = (const RPacket &other)
    {
        if(this == &other)
            return *this;
        m_head = other.m_head;
        m_len = other.m_len;
        m_buf = other.m_buf;
        m_readPos = other.m_readPos;
        if(m_binBuffer)
        {
            delete[] m_binBuffer;
            m_binBufferPos = 0;
        }
        return *this;
    }

    char  ReadChar()
    {
        return Read<char>();
    }

    short ReadShort()
    {
        return Read<short>();
    }

    long  ReadLong()
    {
        return Read<long>();
    }

    float ReadFloat()
    {
        return Read<float>();
    }

    double ReadDouble()
    {
        return Read<double>();
    }

    short ReadCmd()
    {
        return *(short*)&m_buf->m_buf[m_head+sizeOfLen];
    }

    const char* ReadString()
    {
        unsigned int strLen = (unsigned int)Read<short>();
        if(strLen == 0 || m_dataRemain < strLen)
        {
            return 0;
        }
        unsigned int sizeRemain = m_readBuf->m_bufSize - m_readPos;//当前buf还有多少有效数据
        char *str;
        if(sizeRemain >= strLen)
        {    
            str = &m_readBuf->m_buf[m_readPos];
            m_readPos += strLen;
        }
        else
        {

            if(!m_binBuffer)
            {
                m_binBuffer = new char[m_len];
                m_binBufferPos = 0;
            }

            str = &m_binBuffer[m_binBufferPos];
            unsigned int copySize = sizeRemain;
            memcpy(&m_readBuf->m_buf[m_readPos],&m_binBuffer[m_binBufferPos],copySize);
            m_readBuf = m_readBuf->m_next;
            m_readPos = 0;
            m_binBufferPos += copySize;
            copySize = strLen - copySize;
            memcpy(&m_readBuf->m_buf[m_readPos],&m_binBuffer[m_binBufferPos],copySize);
            m_readPos += copySize;
        }
        m_dataRemain -= strLen;
        Arrange();
        return str;
    }
    
    const void* ReadBinary(unsigned short &len)
    {

        len = (unsigned int)Read<short>();
        if(len == 0 || m_dataRemain < len)
        {
            return 0;
        }

        if(m_dataRemain < len)
            return 0;//应提示错误，可能读包顺序与发包不一致了
        
        void *bin;
        
        unsigned int sizeRemain = m_readBuf->m_bufSize - m_readPos;//当前buf还有多少有效数据
        if(sizeRemain >= len)
        {
            bin = &m_readBuf->m_buf[m_readPos];
            m_readPos += len;
        }
        else
        {

            if(!m_binBuffer)
            {
                m_binBuffer = new char[m_len];
                m_binBufferPos = 0;
            }

            bin = &m_binBuffer[m_binBufferPos];

            unsigned int copySize = sizeRemain;
            memcpy(&m_readBuf->m_buf[m_readPos],&m_binBuffer[m_binBufferPos],copySize);
            m_readBuf = m_readBuf->m_next;
            m_readPos = 0;
            m_binBufferPos += copySize;
            copySize = len - copySize;
            memcpy(&m_readBuf->m_buf[m_readPos],&m_binBuffer[m_binBufferPos],copySize);
            m_readPos += copySize;
        }
        m_dataRemain -= len;
        Arrange();
        return bin;
    }

private:
    template <typename T>
    T Read()
    {

        if(m_readBuf._nil())
            return 0;

        unsigned int TypeSize = sizeof(T);

        if(m_dataRemain < TypeSize)
            return 0;//应提示错误，可能读包顺序与发包不一致了

        T ret = 0;

        unsigned int sizeRemain = MAX_PACKET_SIZE - m_readPos;//当前buf还有多少有效数据
        if(sizeRemain >= TypeSize)
        {
            ret = *(T*)&m_readBuf->m_buf[m_readPos];
            m_readPos += TypeSize;
        }
        else
        {
            //数据跨越了两个数据块
            char tmp[sizeof(T)];
            char *ptr = tmp;
            unsigned int copySize = sizeRemain;
            memcpy(&m_readBuf->m_buf[m_readPos],ptr,copySize);
            ptr += copySize;
            copySize = TypeSize - copySize;
            m_readBuf = m_readBuf->m_next;
            m_readPos = 0;
            memcpy(&m_readBuf->m_buf[m_readPos],ptr,copySize);
            m_readPos += copySize;
            ret = *(T*)&tmp[0];
        }
        m_dataRemain -= TypeSize;
        Arrange();
        return ret;
    }

    RPacket(rptr<buffer> &buf,unsigned short head)
        :m_buf(buf),m_readBuf(buf),m_head(head),m_readPos(head+sizeOfLen+cmdlen)
        ,m_len(0),m_dataRemain(0),m_binBuffer(0),m_binBufferPos(0)
    {
        if(!m_buf._nil())
        {
            m_dataRemain = m_len = *(unsigned short*)&m_buf->m_buf[m_head];
        }
    }

    void Arrange()
    {
        if(m_readPos >= m_readBuf->m_bufSize && m_dataRemain > 0)
        {
            m_readPos = 0;
            m_readBuf = m_readBuf->m_next;
        }
    }

private:
    unsigned short m_readPos;
    unsigned short m_head; //在buf中的起始下标
    unsigned short m_len;  //packet的总长度
    unsigned short m_dataRemain;
    ////////用于处理ReadString,和ReadBin时数据跨越buffer的情况
    char *         m_binBuffer;
    unsigned short m_binBufferPos;
    ////////

    
    rptr<buffer> m_readBuf;//当前readPos所在的buf
    rptr<buffer> m_buf;//存放packet的数据，可能由一组m_buf构成链表
};

如代码所示，数据存放在由m_buf组成的list中，m_head表明属于本RPacket的数据在m_buf中的起始位置.

一个buf块的大小为65535字节。如果包的长度不大，则N个RPacket可以共享同一个buf,如果RPacket的数据
大于65535,则一个RPacket的数据则可能会使用超过一块buf。由同一个套接口收到的所有RPacket,其实际数据
被一个buf链表链接着。


buf由基于引用计数的指针指向，当引用同一个buf块的RPacket都被释放之后，buf也将会被释放。


下面介绍数据接收和解包处理过程:

bool Connection::Recv()
{
        if(m_tail._nil())
        {
            m_CurRecvBuf = m_head = m_tail = new buffer(MAX_PACKET_SIZE);
            m_pos = m_totalDataSize = m_writePos = 0;
        }
        unsigned short bufCount = 0;
        unsigned short RecvSize = MAX_PACKET_SIZE;
        unsigned short freeBufSize = m_CurRecvBuf->m_bufSize - m_writePos;
        m_WRecvBuf[bufCount].buf = &m_CurRecvBuf->m_buf[m_writePos];
        m_WRecvBuf[bufCount].len = freeBufSize;
        RecvSize -= freeBufSize;

        if(RecvSize > 0)
        {
            ++bufCount;
            //不够MAX_PACKET_SIZE
            rptr<buffer> tmp = new buffer(MAX_PACKET_SIZE);
            m_tail->m_next = tmp;
            m_tail = tmp;
            m_WRecvBuf[bufCount].buf = &m_tail->m_buf[0];
            m_WRecvBuf[bufCount].len = RecvSize;
        }
        return ConnectionBase::Recv(m_WRecvBuf,bufCount,(OVERLAPPED*)&m_RecvContext);
}


void Connection::OnRecvComplete(unsigned short dwBytesTransfered)
{
    unsigned short freeBufSize = m_CurRecvBuf->m_bufSize - m_writePos;
    if(freeBufSize >= dwBytesTransfered)
    {
        m_writePos += dwBytesTransfered;
        freeBufSize -= dwBytesTransfered;
        m_CurRecvBuf->m_dataSize += dwBytesTransfered;
        if(freeBufSize <= 0)
        {
            //开辟新空间
            rptr<buffer> tmp = new buffer(MAX_PACKET_SIZE);
            m_tail->m_next = tmp;
            m_tail = tmp;
            m_CurRecvBuf = m_tail;
            m_writePos = 0;
        }
    }
    else
    {
        m_CurRecvBuf->m_dataSize = m_CurRecvBuf->m_bufSize;
        //开辟新空间
        rptr<buffer> tmp = new buffer(MAX_PACKET_SIZE);
        m_tail->m_next = tmp;
        m_tail = tmp;
        m_CurRecvBuf = m_tail;
        m_writePos += (dwBytesTransfered - freeBufSize);
        m_CurRecvBuf->m_dataSize += (dwBytesTransfered - freeBufSize);
    }
    
    m_totalDataSize += dwBytesTransfered;
    //Recv();
}

RPacket Connection::UnPack()
{
    if(!m_head._nil())
        return RPacket(rptr<buffer>(0),0);

    unsigned short packetLen = ReadPacketLen();
    if(packetLen == 0)
    {
        return RPacket(rptr<buffer>(0),0);
    }

    if(m_totalDataSize < packetLen)
        return RPacket(rptr<buffer>(0),0);//没有足够的数据

    //OK,数据充足，返回封包
    RPacket rpk(m_head,(unsigned short)m_pos);

    m_totalDataSize -= packetLen;
    
    //调整m_head和m_pos
    m_pos += packetLen;
    while(m_pos >= MAX_PACKET_SIZE)
    {
        rptr<buffer> tmp = m_head;
        m_head = m_head->m_next;
        if(m_head._nil())
        {
            m_tail = 0;
            m_pos = 0;
        }
        else
            m_pos -= MAX_PACKET_SIZE;
    }
    return rpk;
}

完成例程在OnRecvComplete中，

把接收到的数据拷贝到buf链表中，马上启动新的Recv操作.




后续的解包过程是UnPack函数，其作用就是将buf链表中的数据解包，并将RPacket返回以供应用
层处理，从代码中可以看出，解包过程是没有数据拷贝的，只需要正确设置RPacket中的字段就可以了.