6.5——粘包现象与解决方案
简单远程执行命令程序开发
是时候用户socket干点正事呀，我们来写一个远程执行命令的程序，写一个socket client端在windows端发送指令，一个socket server在Linux端执行命令并返回结果给客户端
执行命令的话，肯定是用我们学过的subprocess模块啦，但注意注意注意：
res=subprocess.Popen(cmd.deocde('utf-8'),shell=subprocess.PIPE,stdout=subprocess.PIPE
命令结果的编码是以当前所在的系统为准的，如果是windows，那么res.stdout.read()读出的就是GBK编码的，在接收端需要用GBK解码，且只能从管道里读一次结果
ssh server
import socket import subprocess ip_port = ('127.0.0.1', 8080) tcp_socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tcp_socket_server.bind(ip_port) tcp_socket_server.listen(5) while True: conn, addr = tcp_socket_server.accept() print('客户端', addr) while True: cmd = conn.recv(1024) if len(cmd) == 0: break print("recv cmd",cmd) res = subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) stderr = res.stderr.read() stdout = res.stdout.read() print("res length",len(stdout)) conn.send(stderr) conn.send(stdout)
ssh client
import socket ip_port = ('127.0.0.1', 8080) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) res = s.connect_ex(ip_port) while True: msg = input('>>: ').strip() if len(msg) == 0: continue if msg == 'quit': break s.send(msg.encode('utf-8')) act_res = s.recv(1024) print(act_res.decode('utf-8'), end='')
粘包的解决办法
问题的根源在于，接收端不知道发送端将要传送的字节流的长度，所以解决粘包的方法就是围绕，如何让发送端在发送数据前，把自己将要发送的字节流总大小让接收端知晓，然后接收端来一个死循环接收完所有数据
普通版
服务器端
import socket,subprocess ip_port=('127.0.0.1',8080) s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(ip_port) s.listen(5) while True: conn,addr=s.accept() print('客户端',addr) while True: msg=conn.recv(1024) if not msg:break res=subprocess.Popen(msg.decode('utf-8'),shell=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE) err=res.stderr.read() if err: ret=err else: ret=res.stdout.read() data_length=len(ret) conn.send(str(data_length).encode('utf-8')) data=conn.recv(1024).decode('utf-8') if data == 'recv_ready': conn.sendall(ret) conn.close()
客户端
import socket,time s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) res=s.connect_ex(('127.0.0.1',8080)) while True: msg=input('>>: ').strip() if len(msg) == 0:continue if msg == 'quit':break s.send(msg.encode('utf-8')) length=int(s.recv(1024).decode('utf-8')) s.send('recv_ready'.encode('utf-8')) send_size=0 recv_size=0 data=b'' while recv_size < length: data+=s.recv(1024) recv_size+=len(data) #为什么不直接写1024？ print(data.decode('utf-8'))
为何low？
程序的运行速度远快于网络传输速度，所以在发送一段字节前，先用send去发送该字节流长度，这种方式会放大网络延迟带来的性能损耗
刚才上面 在发送消息之前需先发送消息长度给对端，还必须要等对端返回一个ready收消息的确认，不等到对端确认就直接发消息的话，还是会产生粘包问题(承载消息长度的那条消息和消息本身粘在一起)。 有没有优化的好办法么？
文艺版
思考一个问题，为什么不能在发送了消息长度(称为消息头head吧)给对端后，立刻发消息内容(称为body吧)，是因为怕head 和body 粘在一起，所以通过等对端返回确认来把两条消息中断开。
可不可以直接发head + body,但又能让对端区分出哪个是head，哪个是body呢？我靠、我靠，感觉智商要涨了。
想到了，把head设置成定长的呀，这样对端只要收消息时，先固定收定长的数据，head里写好，后面还有多少是属于这条消息的数据，然后直接写个循环收下来不就完了嘛！唉呀妈呀，我真机智。
可是、可是如何制作定长的消息头呢？假设你有2条消息要发送，第一条消息长度是 3000个字节，第2条消息是200字节。如果消息头只包含消息长度的话，那两个消息的消息头分别是
len(msf1)=4000=4字节
len(msg2)=200=3字节
你的服务端如何完整的收到这个消息头呢？是recv(3)还是recv(4)服务器端怎么知道？用尽我所有知识，我只能想到拼接字符串的办法了，打比方就是设置消息头固定100字节长，不够的拿空字符串去拼接。
server
import socket,json import subprocess ip_port = ('127.0.0.1', 8080) tcp_socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) tcp_socket_server.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) #一行代码搞定，写在bind之前 tcp_socket_server.bind(ip_port) tcp_socket_server.listen(5) def pack_msg_header(header,size): bytes_header = bytes(json.dumps(header),encoding="utf-8") fill_up_size = size - len(bytes_header) print("need to fill up ",fill_up_size) header['fill'] = header['fill'].zfill(fill_up_size) print("new header",header) bytes_new_header = bytes(bytes(json.dumps(header),encoding="utf-8")) return bytes_new_header while True: conn, addr = tcp_socket_server.accept() print('客户端', addr) while True: cmd = conn.recv(1024) if len(cmd) == 0: break print("recv cmd",cmd) res = subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) stderr = res.stderr.read() stdout = res.stdout.read() print("res length",len(stdout)) msg_header = { 'length':len(stdout + stderr), 'fill':'' } packed_header = pack_msg_header(msg_header,100) print("packed header size",packed_header,len(packed_header)) conn.send(packed_header) conn.send(stdout + stderr)
client
import socket import json ip_port = ('127.0.0.1', 8080) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) res = s.connect_ex(ip_port) while True: msg = input('>>: ').strip() if len(msg) == 0: continue if msg == 'quit': break s.send(msg.encode('utf-8')) response_msg_header = s.recv(100).decode("utf-8") response_msg_header_data = json.loads(response_msg_header) msg_size = response_msg_header_data['length'] res = s.recv(msg_size) print("received res size ",len(res)) print(res.decode('utf-8'), end='')
文艺版二
为字节流加上自定义固定长度报头也可以借助第三方模块struct，用法为
import json,struct #假设通过客户端上传1T:1073741824000的文件a.txt #为避免粘包,必须自定制报头 header={'file_size':1073741824000,'file_name':'/a/b/c/d/e/a.txt','md5':'8f6fbf8347faa4924a76856701edb0f3'} #1T数据,文件路径和md5值 #为了该报头能传送,需要序列化并且转为bytes head_bytes=bytes(json.dumps(header),encoding='utf-8') #序列化并转成bytes,用于传输 #为了让客户端知道报头的长度,用struck将报头长度这个数字转成固定长度:4个字节 head_len_bytes=struct.pack('i',len(head_bytes)) #这4个字节里只包含了一个数字,该数字是报头的长度 #客户端开始发送 conn.send(head_len_bytes) #先发报头的长度,4个bytes conn.send(head_bytes) #再发报头的字节格式 conn.sendall(文件内容) #然后发真实内容的字节格式 #服务端开始接收 head_len_bytes=s.recv(4) #先收报头4个bytes,得到报头长度的字节格式 x=struct.unpack('i',head_len_bytes)[0] #提取报头的长度 head_bytes=s.recv(x) #按照报头长度x,收取报头的bytes格式 header=json.loads(json.dumps(header)) #提取报头 #最后根据报头的内容提取真实的数据,比如 real_data_len=s.recv(header['file_size']) s.recv(real_data_len)
使用struct模块实现方式如下
server
import socket,struct,json import subprocess phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) #就是它，在bind前加 phone.bind(('127.0.0.1',8080)) phone.listen(5) while True: conn,addr=phone.accept() while True: cmd=conn.recv(1024) if not cmd:break print('cmd: %s' %cmd) res=subprocess.Popen(cmd.decode('utf-8'), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) err=res.stderr.read() print(err) if err: back_msg=err else: back_msg=res.stdout.read() headers={'data_size':len(back_msg)} head_json=json.dumps(headers) head_json_bytes=bytes(head_json,encoding='utf-8') conn.send(struct.pack('i',len(head_json_bytes))) #先发报头的长度 conn.send(head_json_bytes) #再发报头 conn.sendall(back_msg) #在发真实的内容 conn.close()
client
from socket import * import struct,json ip_port=('127.0.0.1',8080) client=socket(AF_INET,SOCK_STREAM) client.connect(ip_port) while True: cmd=input('>>: ') if not cmd:continue client.send(bytes(cmd,encoding='utf-8')) head=client.recv(4) #先收4个bytes，这里4个bytes里包含了报头的长度 head_json_len=struct.unpack('i',head)[0] #解出报头的长度 head_json=json.loads(client.recv(head_json_len).decode('utf-8')) #拿到报头 data_len=head_json['data_size'] #取出报头内包含的信息 #开始收数据 recv_size=0 recv_data=b'' while recv_size < data_len: recv_data+=client.recv(1024) recv_size+=len(recv_data) print(recv_data.decode('utf-8'))
6.6——通过socket发送文件
通过socket收发文件软件开发
1、客户端提交命令 2、服务端接收命令，解析，执行下载文件的方法，即以读的方式打开文件，for循环读出文件的一行行内容，然后send给客户端 3、客户端以写的方式打开文件，将接收的内容写入文件中
参照上一小节文艺青年实现版二，示范代码如下
服务端实现
import socket import struct import json import subprocess import os class MYTCPServer: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM allow_reuse_address = False max_packet_size = 8192 coding='utf-8' request_queue_size = 5 server_dir='file_upload' def __init__(self, server_address, bind_and_activate=True): """Constructor. May be extended, do not override.""" self.server_address=server_address self.socket = socket.socket(self.address_family, self.socket_type) if bind_and_activate: try: self.server_bind() self.server_activate() except: self.server_close() raise def server_bind(self): """Called by constructor to bind the socket. """ if self.allow_reuse_address: self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() def server_activate(self): """Called by constructor to activate the server. """ self.socket.listen(self.request_queue_size) def server_close(self): """Called to clean-up the server. """ self.socket.close() def get_request(self): """Get the request and client address from the socket. """ return self.socket.accept() def close_request(self, request): """Called to clean up an individual request.""" request.close() def run(self): while True: self.conn,self.client_addr=self.get_request() print('from client ',self.client_addr) while True: try: head_struct = self.conn.recv(4) if not head_struct:break head_len = struct.unpack('i', head_struct)[0] head_json = self.conn.recv(head_len).decode(self.coding) head_dic = json.loads(head_json) print(head_dic) #head_dic={'cmd':'put','filename':'a.txt','filesize':123123} cmd=head_dic['cmd'] if hasattr(self,cmd): func=getattr(self,cmd) func(head_dic) except Exception: break def put(self,args): file_path=os.path.normpath(os.path.join( self.server_dir, args['filename'] )) filesize=args['filesize'] recv_size=0 print('----->',file_path) with open(file_path,'wb') as f: while recv_size < filesize: recv_data=self.conn.recv(self.max_packet_size) f.write(recv_data) recv_size+=len(recv_data) print('recvsize:%s filesize:%s' %(recv_size,filesize)) tcpserver1=MYTCPServer(('127.0.0.1',8080)) tcpserver1.run()
客户端
import socket import struct import json import os class MYTCPClient: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM allow_reuse_address = False max_packet_size = 8192 coding='utf-8' request_queue_size = 5 def __init__(self, server_address, connect=True): self.server_address=server_address self.socket = socket.socket(self.address_family, self.socket_type) if connect: try: self.client_connect() except: self.client_close() raise def client_connect(self): self.socket.connect(self.server_address) def client_close(self): self.socket.close() def run(self): while True: inp=input(">>: ").strip() if not inp:continue l=inp.split() cmd=l[0] if hasattr(self,cmd): func=getattr(self,cmd) func(l) def put(self,args): cmd=args[0] filename=args[1] if not os.path.isfile(filename): print('file:%s is not exists' %filename) return else: filesize=os.path.getsize(filename) head_dic={'cmd':cmd,'filename':os.path.basename(filename),'filesize':filesize} print(head_dic) head_json=json.dumps(head_dic) head_json_bytes=bytes(head_json,encoding=self.coding) head_struct=struct.pack('i',len(head_json_bytes)) self.socket.send(head_struct) self.socket.send(head_json_bytes) send_size=0 with open(filename,'rb') as f: for line in f: self.socket.send(line) send_size+=len(line) print(send_size) else: print('upload successful') client=MYTCPClient(('127.0.0.1',8080)) client.run()