1024个读出线程的测试结果

1024个读出线程，发送端不控制速度。测试包长2KB, 测试时间983秒时， 单个线程的带宽为0.0068Gb/s, 此时的拥塞窗口为17，rtt为45000. 发送端的cpu占用率为18%， 接收端的CPU占用率为34%。接收端的Recv-Q为0，发送端的Send-Q为几百KB。

发送端和接收端先建立1024个连接，然后再用pthread_create创建1024个线程，发送端每个线程调用send（）函数发送数据，接收端每个线程调用recv（）函数接收数据。

在发送数据的初期，单线程的带宽达到200Mb/s,随着每个线程发送到网络的分组越来越多，网络发生了拥塞，tcp的重传率明显提高到2%, 接着拥塞窗口不断降低，带宽也降低到一个很小的值，接着会发生慢启动的过程，带宽又逐渐上升，并维持在每个线程6Mb/s左右, 总带宽大约为6Gb/s左右。

测试程序：

服务器端：g++ -lpthread server1bak.c -o sbak

1 #include <stdio.h>
  2 #include <stdlib.h>
  3 
  4 #include<pthread.h>
  5 #include <unistd.h>
  6 
  7 #include <sys/socket.h>
  8 #include <arpa/inet.h>
  9 #include <netinet/in.h>
 10 #include <netinet/tcp.h>
 11 #include<sys/time.h>
 12 #include<errno.h>
 13 #include<string.h>
 14 
 15 #define IP "192.168.250.146"
 16 #define PORT  33333
 17 #define SOCKNUM 1024
 18 #define PACKETSIZE 2048
 19 
 20 typedef struct {
 21   int sock;
 22 }ARG;
 23 
 24 pthread_t tid[SOCKNUM];
 25 
 26 int senddata(int sock)
 27 {
 28   int ret;
 29   char temp[PACKETSIZE+1] = "I want to know!";
 30   int size_left=2048;
 31   while( size_left > 0)
 32   {
33     ret = send(sock, temp, size_left, 0);
 34     if (ret < 0)
 35     {
 36       perror("send fail");
 37       exit(1);
 38     }
 39     size_left -= ret;
 40   }
 41 
 42   return size_left;
 43 }
 44 
 45 
 46 int read_cwnd(int tcp_work_socket)
 47 {
 48   struct tcp_info info;
 49   socklen_t  tcp_info_length = sizeof(tcp_info);
 50   if ( getsockopt(tcp_work_socket, SOL_TCP, TCP_INFO, (void *)&info, &tcp_info_length) == 0 ) {
 51     printf(" cwnd:%u, snd_ssthresh:%u, rtt:%u, rtt_d:%u
",
 52         info.tcpi_snd_cwnd,
 53         info.tcpi_snd_ssthresh,
 54         info.tcpi_rtt,
 55         info.tcpi_rttvar
 56        );
 57   }
 58   return 0;
 59 }
 60 
 61 
 62 void *sendData(void *arg)
63 {
 64 #if 1 
 65   ARG *a = (ARG *)arg;
 66   int accept_sock = a->sock;
 67 
 68   long count = 0;
 69   struct  timeval  start;
 70   struct  timeval  end;
 71   unsigned long timer=0;
 72   gettimeofday(&start,NULL);
 73 
 74 
 75   while(1){
 76     senddata(accept_sock);
 77     //usleep(850);
 78     if(pthread_self()== tid[0])
 79     {
 80 //    FILE* fpointer = fopen("result.out", "a+");
 81     count++;
 82     gettimeofday(&end,NULL);
 83     timer = 1000000 * (end.tv_sec-start.tv_sec)+ end.tv_usec-start.tv_usec;
 84     if(timer%1000== 0)
 85     {
 86       printf("count: %ld, socket: %ld,  %lf s, %lf Gb/s, ", count, accept_sock,  timer/1000000.0, count*2048.0/timer/1024*8);
 87       read_cwnd(accept_sock);
 88     }
 89 
 90 #if 1 
 91     if(timer > 3600000000)
 92     {
 93       printf("before close: timer: %ld
", timer);
 94       close(accept_sock);
 95 //      fclose(fpointer);
 96       break;
 97     }
 98 #endif
 99     }
100   }
101   return 0;
102 #endif
103 
104 }
105 
106 
107 
108 int main()
109 {
110   int accept_sock[SOCKNUM];
111   struct sockaddr_in addr_ser;
112 
113 
114   int sock = socket(AF_INET, SOCK_STREAM, 0);
115     if (sock < 0)
116   {
117     perror("create sock fail");
118     exit(1);
119   }
120 
121     addr_ser.sin_family = AF_INET;
122   addr_ser.sin_port = htons(PORT);
123   addr_ser.sin_addr.s_addr = inet_addr(IP);
124 
125 
126     int sockopt = 1;
127   if (setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, (char *)&sockopt, sizeof(int)) < 0)
128   {
129     perror("setsockopt fail");
130     exit(1);
131   }
132 
133   if (bind(sock, (struct sockaddr*)&addr_ser, sizeof(struct sockaddr)) < 0)
134   {
135     perror("bind fail");
136     exit(1);
137   }
138 
139   if (listen(sock, 2000) < 0)
140   {
141     perror("listen fail");
142     exit(1);
143   }
144 
145 
146    for(int i=0; i<SOCKNUM; i++)
147    {
148 
149   accept_sock[i] = accept(sock, 0, 0);
150   if (accept_sock[i] < 0)
151   {
152     perror("accept fail");
153     exit(1);
154   }
155   printf("accept ok!
");
156    }
157 
158 
159 #if 1 
160 
161    //static extern  pthread_t tid[SOCKNUM];
162    ARG a[SOCKNUM];
163    for(int i=0; i<SOCKNUM; i++){
164    a[i].sock = accept_sock[i];
165    pthread_create(&tid[i], 0, sendData, (void *)&a[i]);
166    }
167 #endif
168 
169 #if 1 
170    for(int i=0; i<SOCKNUM; i++)
171    {
172      pthread_join(tid[i], 0);
173    }
174 #endif
175 
176    return 0;
177 }

客户端： g++ -lpthread client1_multi_sock.c -o cbak

 1 #include <stdio.h>
  2 #include <stdlib.h>
  3 
  4 #include<pthread.h>
  5 #include <unistd.h>
  6 #include <time.h>
  7 
  8 #include <sys/socket.h>
  9 #include <arpa/inet.h>
 10 #include <netinet/in.h>
 11 #include<sys/time.h>
 12 #include<string>
 13 
 14 #define IP "192.168.250.146"
 15 #define PORT 33333
 16 
 17 #define SOCKNUM 1024 
 18 #define THREAD_NUM 1024 
 19 #define SOCKET_PER_THREAD 1
 20 
 21 
 22 typedef struct{
 23   int sock[SOCKET_PER_THREAD];
 24 }ARG;
 25 
 26 
 27 int recvdata(int sock, char *buffer)
 28 {
 29   int msgsize = 2048;
 30   int ret;
 31   int nrecv=0;
 32   while (nrecv < msgsize)
33   {
 34     ret = recv(sock, buffer, msgsize-nrecv, 0);
 35     if (ret < 0)
 36     {
 37       perror("recv fail");
 38       exit(1);
 39     }
 40     else
 41     {
 42       nrecv += ret;
 43     }
 44   }
 45   return nrecv;
 46 }
 47 
 48 void *recvData(void *arg)
 49 {
 50   ARG* a = (ARG*)arg;
 51   int *socket = a->sock;
 52   char buffer[2048] = "0";
 53   int count = 0;
 54   struct  timeval  start;
 55   struct  timeval  end;
 56   unsigned long timer;
 57   gettimeofday(&start,NULL);
 58 
 59   while(1)
 60   {
 61     for(int i=0; i<SOCKET_PER_THREAD; i++)
 62     {
 63       recvdata(socket[i], buffer);
 64 #if 0
 65       count++;
 66       gettimeofday(&end,NULL);
 67       timer = 1000000 * (end.tv_sec-start.tv_sec)+ end.tv_usec-start.tv_usec;
 68       if(timer % 5000000==0)
 69       {
 70         printf("timer = %ld us, %lf Gb/s
",timer, count*2048.0/timer/1024*8);
 71       }
 72 #endif
 73     }
 74   }
 75   return 0;
 76 }
 77 
 78 
 79 int main()
 80 {
 81   int sock[SOCKNUM];
 82   int port[SOCKNUM];
 83   struct sockaddr_in addr_ser;
 84   struct sockaddr_in addr_cli[SOCKNUM];
 85 
 86   std::string local_ip("192.168.250.141");
 87 
 88   for(int i=0; i<SOCKNUM; i++)
 89   {
 90     sock[i] = socket(AF_INET, SOCK_STREAM, 0);
 91     if(sock[i] < 0)
 92     {
93       printf("%d ", i);
 94       perror("create socket fail");
 95     }
 96 
 97     addr_ser.sin_family = AF_INET;
 98     addr_ser.sin_port = htons(PORT);
 99     addr_ser.sin_addr.s_addr = inet_addr(IP);
100 
101     addr_cli[i].sin_family = AF_INET;
102     addr_cli[i].sin_port = htons(12345+i);
103     addr_cli[i].sin_addr.s_addr = inet_addr(local_ip.c_str());
104 
105 
106 
107     int sockopt = 1;
108     if (setsockopt(sock[i], SOL_SOCKET, SO_REUSEADDR, (char*)&sockopt, sizeof(sockopt)) == -1)
109     {
110       perror("set reuse fail ");
111       exit(1);
112     }
113 
114     if( bind(sock[i], (struct sockaddr*)&addr_cli[i], sizeof(addr_cli[i]) ) < 0 )
115     {
116       perror("TCP bind: ");
117       exit(1);
118     }
119     printf("bind ok!
");
120 
121     if(connect(sock[i], (struct sockaddr*)&addr_ser, sizeof(struct sockaddr)) < 0)
122     {
123       perror("connect fail:");
124       exit(1);
125     }
126     printf("connect ok!
");
127 
128   }
129 
130 
131   pthread_t tid[THREAD_NUM];
132   ARG a[THREAD_NUM];
133   for(int i=0; i<THREAD_NUM; i++)
134   {
135     for(int j=0; j<SOCKET_PER_THREAD; j++)
136     {
137     a[i].sock[j] = sock[i*SOCKET_PER_THREAD+j];
138     }
139     pthread_create(&tid[i], 0, recvData, (void *)&a[i]);
140   }
141 
142   for(int i=0; i<SOCKNUM; i++)
143   {
144     pthread_join(tid[i], 0);
145   }
146 
147   return 0;
148 }