七,struct模块
解c语言的人,一定会知道struct结构体在c语言中的作用,它定义了一种结构,里面包含不同类型的数据(int,char,bool等等),方便对某一结构对象进行处理。而在网络通信当中,大多传递的数据是以二进制流(binary data)存在的。当传递字符串时,不必担心太多的问题,而当传递诸如int、char之类的基本数据的时候,就需要有一种机制将某些特定的结构体类型打包成二进制流的字符串然后再网络传输,而接收端也应该可以通过某种机制进行解包还原出原始的结构体数据。python中的struct模块就提供了这样的机制,该模块的主要作用就是对python基本类型值与用python字符串格式表示的C struct类型间的转化(This module performs conversions between Python values and C structs represented as Python strings.)。stuct模块提供了很简单的几个函数,下面写几个例子。
1,基本的pack和unpack
struct提供用format specifier方式对数据进行打包和解包(Packing and Unpacking)。例如:
#该模块可以把一个类型,如数字,转成固定长度的bytes类型
import struct
# res = struct.pack('i',12345)
# print(res,len(res),type(res)) #长度是4
res2 = struct.pack('i',12345111)
print(res2,len(res2),type(res2)) #长度也是4
unpack_res =struct.unpack('i',res2)
print(unpack_res) #(12345111,)
# print(unpack_res[0]) #12345111
代码中,首先定义了一个元组数据,包含int、string、float三种数据类型,然后定义了struct对象,并制定了format‘I3sf’,I 表示int,3s表示三个字符长度的字符串,f 表示 float。最后通过struct的pack和unpack进行打包和解包。通过输出结果可以发现,value被pack之后,转化为了一段二进制字节串,而unpack可以把该字节串再转换回一个元组,但是值得注意的是对于float的精度发生了改变,这是由一些比如操作系统等客观因素所决定的。打包之后的数据所占用的字节数与C语言中的struct十分相似。
2,定义format可以参照官方api提供的对照表:
3,基本用法
import json,struct
#假设通过客户端上传1T:1073741824000的文件a.txt
#为避免粘包,必须自定制报头
header={'file_size':1073741824000,'file_name':'/a/b/c/d/e/a.txt','md5':'8f6fbf8347faa4924a76856701edb0f3'} #1T数据,文件路径和md5值
#为了该报头能传送,需要序列化并且转为bytes
head_bytes=bytes(json.dumps(header),encoding='utf-8') #序列化并转成bytes,用于传输
#为了让客户端知道报头的长度,用struck将报头长度这个数字转成固定长度:4个字节
head_len_bytes=struct.pack('i',len(head_bytes)) #这4个字节里只包含了一个数字,该数字是报头的长度
#客户端开始发送
conn.send(head_len_bytes) #先发报头的长度,4个bytes
conn.send(head_bytes) #再发报头的字节格式
conn.sendall(文件内容) #然后发真实内容的字节格式
#服务端开始接收
head_len_bytes=s.recv(4) #先收报头4个bytes,得到报头长度的字节格式
x=struct.unpack('i',head_len_bytes)[0] #提取报头的长度
head_bytes=s.recv(x) #按照报头长度x,收取报头的bytes格式
header=json.loads(json.dumps(header)) #提取报头
#最后根据报头的内容提取真实的数据,比如
real_data_len=s.recv(header['file_size'])
s.recv(real_data_len)
FTP作业:上传下载文件
服务端:
import socket
import struct
import json
import subprocess
import os
class MYTCPServer:
address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
allow_reuse_address = False
max_packet_size = 8192
coding='utf-8'
request_queue_size = 5
server_dir='file_upload'
def __init__(self, server_address, bind_and_activate=True):
"""Constructor. May be extended, do not override."""
self.server_address=server_address
self.socket = socket.socket(self.address_family,
self.socket_type)
if bind_and_activate:
try:
self.server_bind()
self.server_activate()
except:
self.server_close()
raise
def server_bind(self):
"""Called by constructor to bind the socket.
"""
if self.allow_reuse_address:
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
self.server_address = self.socket.getsockname()
def server_activate(self):
"""Called by constructor to activate the server.
"""
self.socket.listen(self.request_queue_size)
def server_close(self):
"""Called to clean-up the server.
"""
self.socket.close()
def get_request(self):
"""Get the request and client address from the socket.
"""
return self.socket.accept()
def close_request(self, request):
"""Called to clean up an individual request."""
request.close()
def run(self):
while True:
self.conn,self.client_addr=self.get_request()
print('from client ',self.client_addr)
while True:
try:
head_struct = self.conn.recv(4)
if not head_struct:break
head_len = struct.unpack('i', head_struct)[0]
head_json = self.conn.recv(head_len).decode(self.coding)
head_dic = json.loads(head_json)
print(head_dic)
#head_dic={'cmd':'put','filename':'a.txt','filesize':123123}
cmd=head_dic['cmd']
if hasattr(self,cmd):
func=getattr(self,cmd)
func(head_dic)
except Exception:
break
def put(self,args):
file_path=os.path.normpath(os.path.join(
self.server_dir,
args['filename']
))
filesize=args['filesize']
recv_size=0
print('----->',file_path)
with open(file_path,'wb') as f:
while recv_size < filesize:
recv_data=self.conn.recv(self.max_packet_size)
f.write(recv_data)
recv_size+=len(recv_data)
print('recvsize:%s filesize:%s' %(recv_size,filesize))
tcpserver1=MYTCPServer(('127.0.0.1',8080))
tcpserver1.run()
客户端
import socket
import struct
import json
import os
class MYTCPClient:
address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
allow_reuse_address = False
max_packet_size = 8192
coding='utf-8'
request_queue_size = 5
def __init__(self, server_address, connect=True):
self.server_address=server_address
self.socket = socket.socket(self.address_family,
self.socket_type)
if connect:
try:
self.client_connect()
except:
self.client_close()
raise
def client_connect(self):
self.socket.connect(self.server_address)
def client_close(self):
self.socket.close()
def run(self):
while True:
inp=input(">>: ").strip()
if not inp:continue
l=inp.split()
cmd=l[0]
if hasattr(self,cmd):
func=getattr(self,cmd)
func(l)
def put(self,args):
cmd=args[0]
filename=args[1]
if not os.path.isfile(filename):
print('file:%s is not exists' %filename)
return
else:
filesize=os.path.getsize(filename)
head_dic={'cmd':cmd,'filename':os.path.basename(filename),'filesize':filesize}
print(head_dic)
head_json=json.dumps(head_dic)
head_json_bytes=bytes(head_json,encoding=self.coding)
head_struct=struct.pack('i',len(head_json_bytes))
self.socket.send(head_struct)
self.socket.send(head_json_bytes)
send_size=0
with open(filename,'rb') as f:
for line in f:
self.socket.send(line)
send_size+=len(line)
print(send_size)
else:
print('upload successful')
client=MYTCPClient(('127.0.0.1',8080))
client.run()
文件传输
下载功能是服务端以读的方式打开文件;客户端以写的方式打开文件;
上传功能恰相反,客户端以读的方式打开,服务端以写的方式打开一个新文件接收客户端的传输;
下载功能的实现:
简单版的实现
服务端:
#服务端 import socket import subprocess import struct import json import os share_dir=r'C:UsersAdministratorPycharmProjectsmyFirstprochapter6网络编程文件传输简单版本servershare' phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) # phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) phone.bind(('127.0.0.1',8912)) #0-65535:0-1024给操作系统使用 phone.listen(5) print('starting...') while True: # 链接循环 conn,client_addr=phone.accept() print(client_addr) while True: #通信循环 try: #1、收命令 res=conn.recv(8096) # b'get 3.jpeg' if not res:break #适用于linux操作系统 #2、解析命令,提取相应命令参数 cmds=res.decode('utf-8').split() #['get','3.jpeg'] filename=cmds[1] #3、以读的方式打开文件,读取文件内容发送给客户端 #第一步:制作固定长度的报头 header_dic={ 'filename': filename, #'filename':'3.jpeg' 'md5':'xxdxxx', 'file_size': os.path.getsize(r'%s/%s' %(share_dir,filename)) #os.path.getsize(r'C:UsersAdministratorPycharmProjectsmyFirstprochapter6网络编程文件传输简单版本servershare3.jpeg') } header_json=json.dumps(header_dic) header_bytes=header_json.encode('utf-8') #第二步:先发送报头的长度 conn.send(struct.pack('i',len(header_bytes))) #第三步:再发报头 conn.send(header_bytes) #第四步:再发送真实的数据 with open('%s/%s' %(share_dir,filename),'rb') as f: # conn.send(f.read()) for line in f: conn.send(line) except ConnectionResetError: #适用于windows操作系统 break conn.close() phone.close()
客户端:
##客户端 import socket import struct import json download_dir=r'C:UsersAdministratorPycharmProjectsmyFirstprochapter6网络编程文件传输简单版本clientdownload' phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM) phone.connect(('127.0.0.1',8912)) while True: #1、发命令 cmd=input('>>: ').strip() #get a.txt if not cmd:continue phone.send(cmd.encode('utf-8')) #2、以写的方式打开一个新文件,接收服务端发来的文件的内容写入客户的新文件 #第一步:先收报头的长度 obj=phone.recv(4) header_size=struct.unpack('i',obj)[0] #第二步:再收报头 header_bytes=phone.recv(header_size) #第三步:从报头中解析出对真实数据的描述信息 header_json=header_bytes.decode('utf-8') header_dic=json.loads(header_json) ''' header_dic={ 'filename': filename, ##'filename':'3.jpeg' 'md5':'xxdxxx', 'file_size': os.path.getsize(filename) } ''' print(header_dic) total_size=header_dic['file_size'] filename=header_dic['filename'] #第四步:接收真实的数据 with open('%s/%s' %(download_dir,filename),'wb') as f: #拼接一个绝对路径 recv_size=0 while recv_size < total_size: line=phone.recv(1024) #1024是一个坑 f.write(line) recv_size+=len(line) print('总大小:%s 已下载大小:%s' %(total_size,recv_size)) phone.close()
使用函数进行代码的优化(下载)
服务端:
###服务端 import socket import os import struct import pickle class TCPServer: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM listen_count = 5 max_recv_bytes = 8192 coding = 'utf-8' allow_reuse_address = False #默认不允许重用端口 # 下载的文件存放路径 down_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'share') # 上传的文件存放路径 upload_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'upload') def __init__(self,server_address,bind_and_listen=True): self.server_address = server_address self.socket = socket.socket(self.address_family,self.socket_type) if bind_and_listen: try: self.server_bind() #绑定 self.server_listen() #激活 except Exception: self.server_close() def server_bind(self): if self.allow_reuse_address: self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) self.socket.bind(self.server_address) #self.server_address = self.socket.getsockname() def server_listen(self): self.socket.listen(self.listen_count) def server_close(self): self.socket.close() def server_accept(self): return self.socket.accept() def conn_close(self,conn): conn.close() def run(self): print('starting...') while True: self.conn,self.client_addr = self.server_accept() print(self.client_addr) while True: try: res = self.conn.recv(self.max_recv_bytes) if not res:continue cmds = res.decode(self.coding).split() if hasattr(self,cmds[0]): func = getattr(self,cmds[0]) func(cmds) except Exception: break self.conn_close(self.conn) def get(self,cmds): """ 下载文件 1.找到下载的文件 2.发送 header_size 3.发送 header_bytes file_size 4.读文件 rb 发送 send(line) 5.若文件不存在,发送0 client提示:文件不存在 :param cmds: 下载的文件 eg:['get','3.jpeg'] :return: """ filename = cmds[1] file_path = os.path.join(self.down_filepath, filename) if os.path.isfile(file_path): header = { 'filename': filename, 'md5': 'xxxxxx', 'file_size': os.path.getsize(file_path) } header_bytes = pickle.dumps(header) self.conn.send(struct.pack('i', len(header_bytes))) self.conn.send(header_bytes) with open(file_path, 'rb') as f: for line in f: self.conn.send(line) else: self.conn.send(struct.pack('i', 0)) def put(self,cmds): """ 上传功能 1.接收4个bytes 得到文件的 header_size 2.根据 header_size 得到 header_bytes header_dic 3.根据 header_dic 得到 file_size 3.以写的形式 打开文件 f.write() :param cmds: 下载的文件 eg:['put','Amanda.jpg'] :return: """ obj = self.conn.recv(4) header_size = struct.unpack('i', obj)[0] header_bytes = self.conn.recv(header_size) header_dic = pickle.loads(header_bytes) print(header_dic) file_size = header_dic['file_size'] filename = header_dic['filename'] with open('%s/%s' % (self.upload_filepath, filename), 'wb') as f: recv_size = 0 while recv_size < file_size: res = self.conn.recv(self.max_recv_bytes) f.write(res) recv_size += len(res) tcp_server = TCPServer(('127.0.0.1',8080)) tcp_server.run() tcp_server.server_close()
客户端:
##客户端 import socket import struct import pickle import os class FTPClient: address_family = socket.AF_INET socket_type = socket.SOCK_STREAM # 下载的文件存放路径 down_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'download') # 上传的文件存放路径 upload_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'share') coding = 'utf-8' max_recv_bytes = 8192 def __init__(self, server_address, connect=True): self.server_address = server_address self.socket = socket.socket(self.address_family, self.socket_type) if connect: try: self.client_connect() except Exception: self.client_close() def client_connect(self): self.socket.connect(self.server_address) def client_close(self): self.socket.close() def run(self): while True: # get 3.jpeg 下载 put Amanda.jpg 上传 msg = input(">>>:").strip() if not msg: continue self.socket.send(msg.encode(self.coding)) cmds = msg.split() if hasattr(self,cmds[0]): func = getattr(self,cmds[0]) func(cmds) def get(self, cmds): """ 下载功能 1.得到 header_size 2.得到 header_types header_dic 3.得到 file_size file_name 4.以写的形式打开文件 :param cmds: 下载的内容 eg: cmds = ['get','3.jpeg'] :return: """ obj = self.socket.recv(4) header_size = struct.unpack('i', obj)[0] if header_size == 0: print('文件不存在') else: header_types = self.socket.recv(header_size) header_dic = pickle.loads(header_types) print(header_dic) file_size = header_dic['file_size'] filename = header_dic['filename'] with open('%s/%s' % (self.down_filepath, filename), 'wb') as f: recv_size = 0 while recv_size < file_size: res = self.socket.recv(self.max_recv_bytes) f.write(res) recv_size += len(res) print('总大小:%s 已下载:%s' % (file_size, recv_size)) else: print('下载成功!') def put(self, cmds): """ 上传功能 1.查看上传的文件是否存在 2.上传文件 header_size 3.上传文件 header_bytes 4.以读的形式 打开文件 send(line) :param cmds: 上传的内容 eg: cmds = ['put','a.txt'] :return: """ filename = cmds[1] file_path = os.path.join(self.upload_filepath, filename) if os.path.isfile(file_path): file_size = os.path.getsize(file_path) header = { 'filename': os.path.basename(filename), 'md5': 'xxxxxx', 'file_size': file_size } header_bytes = pickle.dumps(header) self.socket.send(struct.pack('i', len(header_bytes))) self.socket.send(header_bytes) with open(file_path, 'rb') as f: send_bytes = b'' for line in f: self.socket.send(line) send_bytes += line print('总大小:%s 已上传:%s' % (file_size, len(send_bytes))) else: print('上传成功!') else: print('文件不存在') ftp_client = FTPClient(('127.0.0.1',8080)) ftp_client.run() ftp_client.client_close()
一、作业讲解(大文件下载以及进度条展示)
大文件传输:
server.py
import os import json import socket import struct filepath = r'E:BaiduYunDownload[电影天堂www.dy2018.com]移动迷宫3:死亡解药BD国英双语中英双字.mp4' sk = socket.socket() sk.bind(('127.0.0.1',9000)) sk.listen() conn,addr = sk.accept() filename = os.path.basename(filepath) filesize = os.path.getsize(filepath) dic = {'filename':filename,'filesize':filesize} str_dic = json.dumps(dic).encode('utf-8') len_dic = len(str_dic) length = struct.pack('i',len_dic) conn.send(length) # dic的长度 conn.send(str_dic) # dic with open(filepath,'rb') as f: # 文件 while filesize: content = f.read(4096) conn.send(content) filesize -= len(content) ''' 这里不能减等4096,因为文件,最后可能只有3字节。 要根据读取的长度len(content),来计算才是合理的。 ''' conn.close() sk.close()
client.py
import json import struct import socket sk = socket.socket() sk.connect(('127.0.0.1',9000)) dic_len = sk.recv(4) dic_len = struct.unpack('i',dic_len)[0] dic = sk.recv(dic_len) str_dic = dic.decode('utf-8') dic = json.loads(str_dic) with open(dic['filename'],'wb') as f: # 使用wb更严谨一些,虽然可以使用ab while dic['filesize']: content = sk.recv(4096) #这边的4096,可以不用和server对应,改成1024,也可以 dic['filesize'] -= len(content) f.write(content) sk.close()
先执行server.py,再执行client.py
等待30多秒,当前目录就会出现一个视频文件,打开确认,是否可以播放。
客户体验太差了,用户不知道啥时候能接收完,程序到底有没有卡住?下载花了多长时间?都不知道
下面来一个进阶版的,增加进度条和下载时间
主要是修改client.py,代码如下:
import json import struct import socket import sys import time def processBar(num, total): # 进度条 rate = num / total rate_num = int(rate * 100) if rate_num == 100: r = ' %s>%d%% ' % ('=' * rate_num, rate_num,) else: r = ' %s>%d%%' % ('=' * rate_num, rate_num,) sys.stdout.write(r) sys.stdout.flush start_time = time.time() # 开始时间 sk = socket.socket() sk.connect(('127.0.0.1',9000)) dic_len = sk.recv(4) dic_len = struct.unpack('i',dic_len)[0] dic = sk.recv(dic_len) str_dic = dic.decode('utf-8') dic = json.loads(str_dic) with open(dic['filename'],'wb') as f: # 使用wb更严谨一些,虽然可以使用ab content_size = 0 while True: content = sk.recv(4096)<br> f.write(content) # 写入文件 content_size += len(content) # 接收大小 processBar(content_size,dic['filesize']) # 执行进度条函数 if content_size == dic['filesize']:break # 当接收的总大小等于文件大小时,终止循环 sk.close() # 关闭连接 end_time = time.time() # 结束时间 print('本次下载花费了{}秒'.format(end_time - start_time))
执行效果如下:
上面效果展示了100个等号,太长了,那么要缩减到1/3呢?
修改进度条函数
def processBar(num, total): # 进度条 rate = num / total rate_num = int(rate * 100) if rate_num == 100: r = ' %s>%d%% ' % ('=' * int(rate_num / 3), rate_num,) # 控制等号输出数量,除以3,表示显示1/3 else: r = ' %s>%d%%' % ('=' * int(rate_num / 3), rate_num,) sys.stdout.write(r) sys.stdout.flush
再次执行:
再来一个高级版,显示绿色的飞机
代码如下:
def processBar(num, total): # 进度条 rate = num / total rate_num = int(rate * 100) pretty = '✈' if rate_num == 100: r = ' 33[32m{} 33[0m{}% '.format(pretty * int(rate_num / 5), rate_num,) else: r = ' 33[32m{} 33[0m{}%'.format(pretty * int(rate_num / 5), rate_num,) sys.stdout.write(r) sys.stdout.flush
效果如下:
再来一个每秒换色
导入一个随机换色类
import random class Prompt(object): # 提示信息显示 colour_dic = { 'red': 31, 'green': 32, 'yellow': 33, 'blue': 34, 'purple_red': 35, 'bluish_blue': 36, 'white': 37, } def __init__(self): pass @staticmethod def display(msg, colour='white'): choice = Prompt.colour_dic.get(colour) # print(choice) if choice: info = "