一种长连接服务模型实现

模型图如下（event loop + thread pool）(event用epoll，线程同步用队列)

说明：一个中转线程负责连接外部服务器维持tcp连接，接收外部来的请求，转发请求给服务线程处理，服务线程处理完后通知中转线程回复，主要业务代码如下：

1，ProxyThread.cpp

#include "ProxyThread.h"
#include "Singletone.h"
#include "SwitchSvr.h"

TC_Epoller *ProxyThread::_epoller;
map<int, ProxyConnect *> *ProxyThread::_mapSocket;

struct WupProtocol
{
    static int checkRequest(const char *buf, size_t len)
    {
        if(len < 4){
            return 0;
        }
        Tint ti;
        strncpy(ti.byte, buf, 4);
        int bodyLen = ti.integer;
        if(bodyLen <= (int)len - 4){
            return bodyLen;
        }
        return 0;
    }

    static int parseProxy(string &in, string &out)
    {
        try
        {
            int bodylen = checkRequest(in.c_str(), in.length());
            if(bodylen > 0)
            {
                out = in.substr(0, bodylen+4);
                in  = in.substr(bodylen+4);
                return TC_EpollServer::PACKET_FULL;
            }
            else
            {
                return TC_EpollServer::PACKET_LESS;
            }
        }
        catch(exception &ex)
        {
            return TC_EpollServer::PACKET_ERR;
        }

        return TC_EpollServer::PACKET_LESS;
    }

};

void ProxyThread::Init(const string &sConf)
{
    LOG_DEBUG << "ProxyThread::Init succ" << endl;
}

void ProxyThread::Reload()
{
    LOG_DEBUG << "ProxyThread::Reload succ" << endl;
}

void ProxyThread::CreateThread()
{
    pthread_t thread;

    if( !m_bStart )
    {
        m_bStart = true;
        if(pthread_create(&thread, NULL, Run, (void*)this) != 0)
        {
            throw runtime_error("Create ProxyThread fail");
        }
    }
}

int ProxyThread::registerProxy(ProxyConnect *connect)
{
    LOG_DEBUG << "registerProxy start ip:" << connect->ip << " port:" << connect->port << endl;

    char msg[1024] = "connect";
    Tint ti;
    ti.integer = strlen(msg);

    SendData sendData;
    sendData.fd = connect->socket.getfd();
    sendData.buffer.assign(ti.byte, 4);
    sendData.buffer += msg;

    int iRet = SendBuffer(sendData.fd, sendData);
    if(iRet < 0){
        CloseFD(sendData.fd);
        return -1;
    }
    return 0;
}

int ProxyThread::checkSocket(ProxyConnect *connect)
{
    if(!connect->socket.isValid()){
        try{
            int oldfd = connect->socket.getfd();
            connect->socket.createSocket(SOCK_STREAM, AF_INET);
            connect->socket.setblock(false);
            connect->socket.setKeepAlive();
            connect->first = true;
            connect->_recvbuffer.clear();
            connect->_sendbuffer.clear();
            try{
                connect->socket.connect(connect->ip, connect->port);
            }
            catch(TC_SocketConnect_Exception &ex)
            {
                //connect->socket.close();
                if(errno != EINPROGRESS)
                {
                    connect->socket.close();
                    LOG_ERROR << "Connect Proxy " << connect->ip << ":" << connect->port << " Exception:" << errno << endl;

                    return -1;
                }
            }

            if(errno != EINPROGRESS)
            {
                connect->socket.close();
                return -1;
            }

            if(oldfd != connect->socket.getfd()){
                _mapSocket->erase(oldfd);
            }
            _mapSocket->insert(pair<int, ProxyConnect *>(connect->socket.getfd(), connect));

            LOG_DEBUG << "proxy connected fd:" << connect->socket.getfd() << " errno:" << errno << endl;

            _epoller->add(connect->socket.getfd(), connect->socket.getfd(), EPOLLIN);
            registerProxy(connect);
        }
        catch(TC_Socket_Exception &ex)
        {
            LOG_ERROR << "Connect Proxy Socket Error:" << string(ex.what()) << endl;
            connect->socket.close();
            return -1;
        }
        catch(...)
        {
            LOG_ERROR << "Connect Proxy Unknown Error:" << errno << endl;
            return -1;
        }
    }
    return 0;
}

void ProxyThread::ConnectProxy()
{
    LOG_DEBUG << "ProxyThread::ConnectProxy Begin" << endl;
    _epoller = new TC_Epoller(false);
    _epoller->create(100);

    TC_Config conf;
    conf.parseFile(ServerConfig::BasePath + ServerConfig::ServerName + ".conf");
    vector<string> proxys = conf.getDomainVector("/Config/Proxys");
    for(unsigned int i = 0;i < proxys.size();i ++){
        ProxyConnect *connect = new ProxyConnect();
        connect->ip = conf.get("/Config/Proxys/" + proxys[i] + "<ip>", "127.0.0.1");
        connect->port = atoi(conf.get("/Config/Proxys/" + proxys[i] + "<port>", "9999").c_str());
        checkSocket(connect);
    }
    LOG_DEBUG << "ProxyThread::ConnectProxy End" << endl;
}

int ProxyThread::ParseProtocol(string &_recv, int fd, bool &first){
    try
    {
        while(true){
            string ro;
            LOG_DEBUG << "ProxyThread:ParseProtocol:" << recv << endl;
            int b = WupProtocol::parseProxy(_recv, ro);
            if(b == TC_EpollServer::PACKET_LESS)
            {
                break;
            }
            else if(b == TC_EpollServer::PACKET_FULL)
            {
                if(first){
                    LOG_DEBUG << "First Package: " << ro << endl;
                    first = false;
                }
                else
                {
                    if(Singletone::getInstance()->from_queue.size()<10000){ // overloaded
                        RecvData *o = new RecvData();
                        o->fd = fd;
                        o->buffer = ro;
                        Singletone::getInstance()->from_queue.push_back(o);
                        LOG_DEBUG << "Push Message Package: " << ro << endl;
                    }else{
                        LOG_ERROR << "Recv Queue OverLoaded . Drop Message !" <<endl;
                    }
                }
            }
            if(_recv.empty()){
                break;
            }
        }
    }
    catch(exception &ex)
    {
        LOG_ERROR << "recv protocol error: " << string(ex.what()) << endl;
        return -1;
    }
    catch(...)
    {
        LOG_ERROR << "recv protocol error: " << endl;
        return -1;
    }
    return 1;
}

int ProxyThread::RecvBuffer(int fd){
    map<int ,ProxyConnect *>::iterator it = _mapSocket->find(fd);
    if(it != _mapSocket->end()){
        ProxyConnect *connect = it->second;
        char buffer[8192] = "";
        while(true)
        {
            int iBytesReceived = connect->socket.recv((void*)buffer, sizeof(buffer));
            if (iBytesReceived < 0)
            {
                if(errno == EAGAIN)
                {
                    LOG_DEBUG << "Recv Buffer EAGAIN" << endl;
                    break;
                }
                else{
                    LOG_DEBUG << "Recv Buffer ERROR iBytesReceived: " << iBytesReceived << " error:" << errno << endl;
                    perror("error");
                    return -1;//closed
                }
            }
            if(iBytesReceived == 0){
                LOG_DEBUG << "Proxy Closed ip:" << connect->ip << " port:" << connect->port << endl;
                return -1;
            }
            connect->_recvbuffer.append(buffer, iBytesReceived);
        }
        return ParseProtocol(connect->_recvbuffer, fd, connect->first);
    }
    return -1;//not exist
}

int ProxyThread::SendBuffer(int fd, SendData &o)
{
    map<int ,ProxyConnect *>::iterator it = _mapSocket->find(fd);
    if(it != _mapSocket->end()){
        ProxyConnect *connect = it->second;

        if(o.fd == fd){
            connect->_sendbuffer += o.buffer;
        }
        size_t pos = 0;
        size_t sendLen = connect->_sendbuffer.length();
        const char *sendBegin = connect->_sendbuffer.c_str();

        while (pos < sendLen)
        {
            int iBytesSent = 0;
            LOG_DEBUG << "Send Buffer buf:" << sendBegin << " iBytesSent:" << iBytesSent << " pos:" << pos << " sendLen:" << sendLen << endl;
            //iBytesSent = write(fd, (const void*)(sendBegin + pos), sendLen - pos);
            iBytesSent = connect->socket.send((const void*)(sendBegin + pos), sendLen - pos, 0);
            if (iBytesSent < 0)
            {
                if(errno == EAGAIN)
                {
                    LOG_DEBUG << "Send Buffer EAGAIN" << endl;
                    break;
                }
                else
                {
                    LOG_DEBUG << "Send Buffer ERROR iBytesSent: " << iBytesSent << " error:" << errno << endl;
                    return -1;
                }
            }

            pos += iBytesSent;
            if(pos < sendLen)
            {
                break;
            }
        }
        if(pos < sendLen){ //need to send later
            _epoller->mod(connect->socket.getfd(), connect->socket.getfd(), EPOLLIN|EPOLLOUT);
        }
        if(pos > 0){
            connect->_sendbuffer = connect->_sendbuffer.substr(pos);
        }
        LOG_DEBUG << "Send Buffer Finished pos:" << pos << " sendLen:" << sendLen << endl;
        
    }
    return 0;
}

void ProxyThread::CloseFD(int fd){
    _epoller->del(fd, fd, 0);
    map<int ,ProxyConnect *>::iterator it = _mapSocket->find(fd);
    if(it != _mapSocket->end()){
        it->second->socket.close();
        LOG_ERROR << "Close Proxy fd:" << fd << " ip" << it->second->ip << " port:" << it->second->port << endl;
    }
}

void* ProxyThread::Run(void* arg)
{
    pthread_detach(pthread_self());
    ProxyThread* pthis = (ProxyThread*) arg;
    pthis->SetRuning(true);

    _mapSocket = new map<int, ProxyConnect *>();
    ConnectProxy();

    while(pthis->IsStart())
    {
        //check alive
        for(map<int ,ProxyConnect *>::iterator it = _mapSocket->begin();it != _mapSocket->end();it ++)
        {
            checkSocket(it->second);
        }

        //check io
        int iEvNum = _epoller->wait(100);//100ms
        if(iEvNum < 0){
            perror("Epoll Wait Error");
            continue;
        }else if(iEvNum > 0){
            for(int i = 0; i < iEvNum; ++i)
            {
                const epoll_event &ev = _epoller->get(i);
                if (ev.events & EPOLLERR || ev.events & EPOLLHUP)
                {
                    LOG_DEBUG << "epoll EPOLLERR|EPOLLHUP fd:" << ev.data.fd << endl;
                    CloseFD(ev.data.fd);
                    continue;
                }

                if(ev.events & EPOLLIN) //read data
                {
                    LOG_DEBUG << "epoll EPOLLIN fd:" << ev.data.fd << endl;
                    int ret = RecvBuffer(ev.data.fd);
                    if(ret < 0)
                    {
                        CloseFD(ev.data.fd);
                        continue;
                    }
                    _epoller->mod(ev.data.fd, ev.data.fd, EPOLLIN);

                }
                if (ev.events & EPOLLOUT) // write data 
                {
                    LOG_DEBUG << "epoll EPOLLOUT fd:" << ev.data.fd << endl;
                    SendData sendData;
                    sendData.fd = 0;
                    int iRet = SendBuffer(ev.data.fd, sendData);
                    if(iRet < 0){
                        CloseFD(ev.data.fd);
                        continue;
                    }
                }
            }
        }
        while(Singletone::getInstance()->to_queue.size() > 0){
            LOG_DEBUG << "ProxyThread Has Data to send. Data Number:" << Singletone::getInstance()->to_queue.size() << endl;
            SendData *sendData = NULL;
            bool ret = Singletone::getInstance()->to_queue.pop_front(sendData,0);
            if(ret){
                map<int ,ProxyConnect *>::iterator it = _mapSocket->find(sendData->fd);
                if(it != _mapSocket->end()){
                    int iRet = SendBuffer(sendData->fd, *sendData);
                    delete sendData;
                    if(iRet < 0){
                        CloseFD(sendData->fd);
                        continue;
                    }
                }
                break;
            }
        }
    }

    LOG_DEBUG << "thread end" << endl;
    pthis->SetRuning(false);
    pthis->SetStart(false);
    

    return NULL;
}

2，WorkerThread.cpp

#include "WorkerThread.h"
#include "Singletone.h"
#include "SwitchSvr.h"

TC_ThreadPool WorkerThread::tpool;
TC_ThreadLock WorkerThread::l;

int WorkerThread::threadnum;

void WorkerThread::Init(const string &sConf)
{
    LOG_DEBUG << "WorkerThread::Init succ" << endl;
    TC_Config conf;
    conf.parseFile(ServerConfig::BasePath + ServerConfig::ServerName + ".conf");

    threadnum = atoi(conf.get("/Config/WorkerThread/<num>", "4").c_str());
}

void WorkerThread::Reload()
{
    LOG_DEBUG << "WorkerThread::Reload succ" << endl;
}

void WorkerThread::CreateThread()
{
    pthread_t thread;

    if( !m_bStart )
    {
        m_bStart = true;
        if(pthread_create(&thread, NULL, Run, (void*)this) != 0)
        {
            throw runtime_error("Create WorkerThread fail");
        }
    }
}

void handlework(RecvData *o)
{
    LOG_DEBUG << "WorkerThread threadid:" << pthread_self() << " handle " << o->buffer << endl;

    string msg = "Hello"+o->buffer.substr(4);
    Tint ti;
    ti.integer = msg.length();

    if(Singletone::getInstance()->to_queue.size() < 10000){
        SendData *sendData = new SendData();
        sendData->fd = o->fd;
        sendData->buffer.assign(ti.byte, 4);
        sendData->buffer += msg;
        Singletone::getInstance()->to_queue.push_back(sendData);
    }
    else
    {
        LOG_ERROR << "Send Queue OverLoaded . Drop Message !" <<endl;
    }
    delete o;
}

void* WorkerThread::Run(void* arg)
{
    LOG_DEBUG << "WorkerThread Run Start " << pthread_self() << endl;
    pthread_detach(pthread_self());
    WorkerThread* pthis = (WorkerThread*) arg;
    pthis->SetRuning(true);

    sleep(1);
    
    tpool.init(threadnum);
    tpool.start();

    TC_Functor<void, TL::TLMaker<RecvData *>::Result> cmd(handlework);
    while(true){
        while(Singletone::getInstance()->from_queue.size() > 0){

            LOG_DEBUG << "WorkerThread Has Work to Do. Work Number:" << Singletone::getInstance()->from_queue.size() << endl;
            RecvData *o = NULL;
            bool ret = Singletone::getInstance()->from_queue.pop_front(o,0);
            if(ret){
                TC_Functor<void, TL::TLMaker<RecvData *>::Result>::wrapper_type fw(cmd, o);
                tpool.exec(fw);
            }
            break;
        }
        usleep(100);
    }

    tpool.waitForAllDone(-1);
    tpool.stop();

    pthis->SetRuning(false);
    pthis->SetStart(false);
    return NULL;
}

定义的结构体及数据如下：

struct ProxyConnect
{
    TC_Socket socket;
    string ip;
    int port;
    string _recvbuffer;
    string _sendbuffer;
    bool first;
};

#ifndef __SINGLETONE_H__
#define __SINGLETONE_H__

#include <pthread.h>
#include "servant/Application.h"
#include "util/tc_config.h"
#include "util/tc_singleton.h"
#include <wbl/pthread_util.h>
#include <map>

using namespace std;

typedef union{
    int integer;
    char byte[4];
} Tint;

struct RecvData
{
    int fd;
    string buffer;
};

struct SendData
{
    int fd;
    string buffer;
};

typedef TC_ThreadQueue<RecvData *> from_proxy_queue;
typedef TC_ThreadQueue<SendData *> to_proxy_queue;

class Singletone : public TC_Singleton<Singletone>
{
public:
    Singletone(){}
    ~Singletone(){}

    void init();

public:
    map<int, taf::JceCurrentPtr> m_Current;

    from_proxy_queue from_queue;
    to_proxy_queue to_queue;
};