作业：FTP

作业题目: 开发一个支持多用户在线的FTP程序

• 作业需求:

  要求
  1.用户加密认证（已实现）
  2.允许同时多用户登录
  3.每个用户有自己的家目录 ，且只能访问自己的家目录（已实现）
  4.对用户进行磁盘配额，每个用户的可用空间不同
  5.允许用户在ftp server上随意切换目录（已实现）
  6.允许用户查看当前目录下文件（已实现）
  7.允许上传和下载文件，保证文件一致性(md5)（已实现）
  8.文件传输过程中显示进度条（已实现）
  9.附加功能：支持文件的断点续传（已实现）

 FTP实现实现并发，在main模块里进行修改，增加了mytherad.py

1.在之前开发的FTP基础上，开发支持多并发的功能
2.不能使用SocketServer模块，必须自己实现多线程
3.必须用到队列Queue模块，实现线程池
4.允许配置最大并发数，比如允许只有10个并发用户

C:.
│ RAMDE
│ 
│ __init__.py
│
├─ftpclient
│ ..bak
│ ..dat
│ ..dir
│ a.txt
│ client.py -- 客户端
│ __init__.py
│
└─ftpserver
│ __init__.py
│
├─bin
│ server.py ----- 启动文件
│ __init__.py
│
├─conf
│ │ account.ini  ---- 用户数据
│ │ setting.py ----- 配置文件
│ │ __init__.py
│ │
│ └─__pycache__
│ setting.cpython-36.pyc
│ __init__.cpython-36.pyc
│
├─core
│ │ logger.py  ----  日志
│ │ main.py  ----- 主程序
│ │ main_server.py  --- 解析命令
│ │ mythread.py --- 实现线程，定义线程池
│ │
│ └─__pycache__
│ logger.cpython-36.pyc
│ main.cpython-36.pyc
│ main_server.cpython-36.pyc
│ __init__.cpython-36.pyc
│
├─home
│ │ __init__.py
│ │
│ └─alex  ---- 主目录
│ a.txt
│ 
│
└─log ---  日志文件

注：需要在cmd上运行程序，否则会报错，服务端：python server.py start  客户端：python client.py  -s ip地址 -P 端口

server.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
import os
import sys


BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_DIR)

if __name__ == '__main__':
    from core import main_server
    argv_parser = main_server.Mangement(sys.argv)
    argv_parser.execute()

setting.py

#！ /usr/bin/env python3
# -*- coding:utf-8 -*-
import os
import socket
import logging

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HOME_DIR = os.path.join(BASE_DIR,'home')
ACCOUNT_INI = r'%sconfaccount.ini'% BASE_DIR

address_family = socket.AF_INET
socket_type = socket.SOCK_STREAM
max_packet_size = 8096
request_queue_size = 5

LOG_LEVE = logging.INFO

LOG_TYPE = {
    'access':'access.log',
}

LOG_FORMATTER =logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOG_PATH = os.path.join(BASE_DIR,'log')

SERVER_ADDR = '127.0.0.1'
SERVER_PORT = 8080

logger.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

from conf import setting
import logging
import os
from logging import handlers

def logger_log(log_type):
    log = logging.getLogger(log_type)
    log.setLevel(setting.LOG_LEVE)

    file_log = os.path.join(setting.LOG_PATH,setting.LOG_TYPE[log_type])
    fh = handlers.TimedRotatingFileHandler(file_log,when='D',interval=3,encoding='utf-8')
    log.addHandler(fh)

    file = setting.LOG_FORMATTER
    fh.setFormatter(file)
    return log

main.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import socket
import json
import configparser
import hashlib
import struct
import os
import subprocess
import time
from conf import setting
from .logger import logger_log
from .mythread import Mythread
from threading import Thread
import queue
access_log = logger_log('access')

class Ftpserver:
    STATUS_CODE ={
        200: "Passed authentication!",
        201: "Wrong username or password!",
        300: "File does not exist !",
        301: "File exist , and this msg include the file size- !",
        302: "This msg include the msg size!",
        350: "Dir changed !",
        351: "Dir doesn't exist !",
        401: "File exist ,ready to re-send !",
        402: "File exist ,but file size doesn't match!",
        404: "File created successfully！",
        405: "File deleted successfully！",
        406: "File already exists！",
        407: "File does not exist ！",
        500: 'Folder deletion succeeded!',
        501: 'Folder does not exist!'
    }
    def __init__(self,mangement_instantiation):
        self.mangement_instantiation = mangement_instantiation
        self.socket = socket.socket(setting.address_family,setting.socket_type)
        self.socket.bind((setting.SERVER_ADDR,setting.SERVER_PORT))
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.listen(setting.request_queue_size)
        self.pool = Mythread(setting.THREAD_SIZE)
        self.account = self.load_accuont()
        self.user_obj = None
        self.user_client = None
        self.user_disk = None
        self.username = None



    def run(self):
        '''启动FTP服务器'''
        print('start FTPserver on %s:%s'.center(50,'*')%(setting.SERVER_ADDR,setting.SERVER_PORT))
        while True:
            try:
                request,self.addr = self.socket.accept()
                # self.request = request
                print('got ab new connection form %s...'%(self.addr,))
                t = self.pool.get_queue()
                obj = t(target=self.handler,args=(request,))
                obj.start()
            except Exception:
                break

    def load_accuont(self):
        '''读取用户数据'''
        config_obj = configparser.ConfigParser()
        config_obj.read(setting.ACCOUNT_INI)
        print(config_obj.sections())
        return config_obj

    def authentication(self,username,password):
        '''验证用户账户'''
        if username in self.account:
            _password = self.account[username]['password']
            md5_obj = hashlib.md5()
            md5_obj.update(password.encode('utf-8'))
            md5_password = md5_obj.hexdigest()
            if md5_password == _password:
                # print('登陆成功！')
                self.user_obj = self.account[username]
                self.user_obj['home'] = os.path.join(setting.HOME_DIR,username)
                self.user_client = self.user_obj['home']
                self.user_disk = self.account[username]['file_disk']
                access_log.info('%s 成功登陆了FTP服务器,IP地址:%s,'%(username,self.addr))
                return True
            else:
                print('登陆失败')
                access_log.error('%s 尝试登陆FTP服务器,IP地址:%s,'%(username,self.addr))
                return False
        else:
            print('登陆失败！')
            access_log.error('%s 尝试登陆FTP服务器,IP地址:%s,' % (username, self.addr))
            return False
    def handler(self,request):
        '''处理用户交互'''
        while True:
            try:
                raw_data = request.recv(4)
                if not raw_data:
                    self.pool.put_queue()
                data_size = struct.unpack('i',raw_data)[0]
                data_bytes = request.recv(data_size)
                data = json.loads(data_bytes.decode('utf-8'))
                action_type = data.get('action_type')
                if action_type:
                    if hasattr(self,'_%s'%action_type):
                        func = getattr(self,'_%s'%action_type)
                        func(data,request)
            except Exception:
                break


    def send_response(self,status_code,*args,**kwargs):
        '''发送消息给客户端'''
        conn = args[0]
        data = kwargs
        data['status_code'] = status_code
        data['status_msg'] = self.STATUS_CODE[status_code]
        bytes_data = json.dumps(data).encode('utf-8')
        data_struct = struct.pack('i',len(bytes_data))
        conn.send(data_struct)
        conn.send(bytes_data)

    def _auth(self,data,*args,**kwargs):
        '''处理用户认证'''
        conn = args[0]
        if self.authentication(data.get('username'),data.get('password')):
            print('登陆成功')
            self.send_response(200,conn)
        else:
            self.send_response(201,conn)

    def _get(self,data,*args):
        '''接收客户端下载'''
        conn = args[0]
        filename = data.get('filename')
        full_path = os.path.join(self.user_client,filename)
        if os.path.isfile(full_path):
            file_size = os.stat(full_path).st_size
            self.send_response(301,conn,file_size=file_size)
            print('准备发送!')
            with open(full_path,'rb') as f:
                for line in f:
                        conn.send(line)
                else:
                    print('文件发送成功')
                    access_log.info('%s 下载了文件%s,IP地址:%s,'% (self.user_obj['username'],filename,self.addr))
        else:
            self.send_response(300,conn)
            access_log.info('%s 下载文件%s失败,IP地址:%s,' % (self.user_obj['username'],filename,self.addr))

    def _put(self,data,*args):
        conn = args[0]
        filename = data.get('filename')
        full_path = os.path.join(self.user_client, filename)
        if os.path.isfile(full_path):
            filename = '%s.%s'%(full_path,time.time())
        else:
            filename = full_path
        with open(filename,'wb') as f:
            total_size = data.get('file_size')
            recv_size = 0
            while recv_size < total_size:
                recv_data = conn.recv(1024)
                f.write(recv_data)
                recv_size += len(recv_data)
                print('总大小:%s，已接收:%s'%(total_size,recv_size))
            else:
                print('接收成功')
                access_log.info('%s 上传了文件%s,IP地址:%s,' % (self.user_obj['username'], filename,self.addr))

    def _ls(self,data,*args):
        '''查看'''
        conn = args[0]
        cmd_obj = subprocess.Popen('dir %s'%self.user_client,shell=True,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout = cmd_obj.stdout.read()
        stderr = cmd_obj.stderr.read()
        request = stdout + stderr
        if not request:
           request = bytes('此目录无文件！')
        self.send_response(302,conn,cmd_size=len(request))
        conn.sendall(request)

    def _mkdir(self,data,*args):
        '''创建文件夹'''
        conn = args[0]
        filename = data.get('inp')
        full_path = os.path.abspath(os.path.join(self.user_client,filename))
        cmd_obj = subprocess.Popen('mkdir  %s'%(full_path),shell=True,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout = cmd_obj.stdout.read()
        stderr = cmd_obj.stderr.read()
        request = stderr + stdout
        if request:
            self.send_response(406,conn)
        self.send_response(404,conn,cmd_mkdir=len(request))
        conn.sendall(request)
        access_log.info('%s 成功创建文件%s,IP地址:%s,' % (self.user_obj['username'],filename,self.addr))
    def _del(self,data,*args):
        '''删除文件'''
        conn = args[0]
        del_name = data.get('del_name')
        full_path = os.path.abspath(os.path.join(self.user_client,del_name))
        cmd_obj = subprocess.Popen('del  %s'%(full_path),shell=True,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout = cmd_obj.stdout.read()
        stderr = cmd_obj.stderr.read()
        request = stderr + stdout
        if not request:
           self.send_response(407,conn)
        self.send_response(405,conn,del_mkdir=len(request))
        conn.sendall(request)
        access_log.info('%s 成功删除文件%s,IP地址:%s,' % (self.user_obj['username'], del_name, self.addr))

    def _rm(self,data,*args):
        '''删除文件夹'''
        conn = args[0]
        rm_name = data.get('rm_name')
        full_path = os.path.abspath(os.path.join(self.user_client,rm_name))
        cmd_obj = subprocess.Popen('rd %s'%(full_path),shell=True,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout = cmd_obj.stdout.read()
        stderr = cmd_obj.stderr.read()
        request = stderr + stdout
        if not request:
           self.send_response(501,conn)
        self.send_response(500,conn,rm_mkdir=len(request))
        conn.sendall(request)
        access_log.info('%s 成功删除文件%s,IP地址:%s,' % (self.user_obj['username'], rm_name, self.addr))

    def _cd(self,data,*args):
        '''切换目录'''
        conn = args[0]
        target = data.get('target_dir')
        full_path = os.path.abspath(os.path.join(self.user_client,target))
        if os.path.isdir(full_path):
            if full_path.startswith(self.user_obj['home']):
                self.user_client = full_path
                relative_dir = self.user_client.replace(self.user_obj['home'],'')
                self.send_response(350,conn,current_dir=relative_dir)
            else:
                self.send_response(351,conn)
        else:
            self.send_response(351,conn)

    def _re_get(self,data,*args):
        conn = args[0]
        filename = data.get('filename')
        file_size = data.get('file_size')
        full_path = os.path.join(self.user_client,filename.strip('\'))
        if os.path.isfile(full_path):
            if os.path.getsize(full_path) == file_size:
                self.send_response(401,filename=filename)
                with open(full_path,'rb') as f:
                    f.seek(data.get('recveived_size'))
                    for line in f:
                        conn.send(line)
                    else:
                      print('发送完毕！')
            else:
               self.send_response(402,conn)
        else:
            self.send_response(300,conn)

main_server.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

from .main import Ftpserver

class Mangement:

    def __init__(self,sys_argv):
        '''解析指令，调用相应的模块'''
        self.sys_argv = sys_argv
        print(sys_argv)
        self.verify_argv()

    def verify_argv(self):
        '''验证指令是否合法'''
        if len(self.sys_argv) <2:
            self.help_msg()
        cmd = self.sys_argv[1]
        if not hasattr(self,cmd):
            print('输入错误！')
            self.help_msg()

    def help_msg(self):
        msg = '''
        start      start FTP server
        stop       stop FTP server
        restart    restart FTP server      
        '''
        exit(msg)

    def execute(self):
        cmd = self.sys_argv[1]
        func = getattr(self,cmd)
        func()

    def start(self):
        server = Ftpserver(self)
        server.run_forever()

client.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import socket
import struct
import os
import json
import optparse
import shelve

class Myclient:
    def __init__(self):
        self.username = None
        self.terminal_dis = None
        self.current_dir = None
        self.shelve_obj = shelve.open('.')
        parser = optparse.OptionParser()
        parser.add_option("-s","--server", dest="server", help="ftp server ip_addr")
        parser.add_option("-P","--port",type="int", dest="port", help="ftp server port")
        parser.add_option("-u","--username", dest="username", help="username info")
        parser.add_option("-p","--password", dest="password", help="password info")
        self.options , self.args = parser.parse_args()
        self.arvs_verify()
        self.make_connt()

    def arvs_verify(self):
        '''检查命令的合法性'''
        if not self.options.server or not self.options.port:
            print('缺少必要的参数！')

    def make_connt(self):
        '''建立链接'''
        self.sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
        self.sock.connect((self.options.server,self.options.port))

    def get_response(self):
        data = self.sock.recv(4)
        hear_size = struct.unpack('i',data)[0]
        hear_bytes = self.sock.recv(hear_size)
        hear_dic = json.loads(hear_bytes)
        return hear_dic

    def auth(self):
        count = 0
        while count < 3:
            username = input('请输入账号：').strip()
            password = input('请输入密码：').strip()
            auth_info = {
                'action_type':'auth',
                'username':username,
                'password':password,
            }
            hear_json = json.dumps(auth_info)
            hear_bytes = hear_json.encode('utf-8')
            self.sock.send(struct.pack('i', len(hear_bytes)))
            self.sock.send(hear_bytes)
            response = self.get_response()
            if response.get('status_code') == 200:
                self.username = username
                self.terminal_dis = '[%s]>>'%self.username
                self.current_dir = '\'
                print('登陆成功！')
                msg = '''
                      del      删除文件
                      ls       查看当前文件
                      cd       切换目录
                      mkdir    创建文件夹
                      put      上传文件
                      get      下载文件
                      rm        删除文件夹
                  '''
                print(msg)
                return  True
            else:
                print(response.get('status_msg'))
                count += 1

    def file_check(self):
        if  list(self.shelve_obj.keys()):
            print('未下载成功的列表'.center(50,'*'))
            for index,abs_file in enumerate(self.shelve_obj.keys()):
                print(self.shelve_obj[abs_file][1])
                recv_size = os.path.getsize(self.shelve_obj[abs_file][1])
                print(' %s.%s  %s  %s   %s'%(index,abs_file,
                                             self.shelve_obj[abs_file][0],
                                             recv_size,
                                             recv_size/self.shelve_obj[abs_file][0]*100))

            while True:
                choice = input('请选择继续下载的文件：').strip()
                if not choice:continue
                if choice == 'back':break
                if choice.isdigit():
                    choice = int(choice)
                    if choice >=0 and choice <= index:
                        selected_file = list(self.shelve_obj.keys())[choice]
                        already_rec_size = os.path.getsize(self.shelve_obj[selected_file][1])
                        self.send_msg('re_get',file_size=self.shelve_obj[selected_file][0],
                                      filename=selected_file,
                                      recveived_size=already_rec_size)
                        response = self.get_response()
                        new_filename = response.get('filename').strip('\')
                        if response.get('status_code') == 401:
                            local_filename = self.shelve_obj[selected_file][1]
                            with open(local_filename,'ab') as f:
                                total_size = self.shelve_obj[selected_file][0]
                                recv_size = already_rec_size
                                current_percent = int(recv_size / total_size * 100)
                                progress_generator = self.progress_bar(total_size,
                                                                       current_percent,current_percent)
                                progress_generator.__next__()
                                while recv_size < total_size:
                                    res = self.sock.recv(1024)
                                    f.write(res)
                                    recv_size += len(res)
                                    progress_generator.send(recv_size)
                                else:
                                    print('下载完毕')
                                    del self.shelve_obj[selected_file]
                            os.rename(local_filename, new_filename)

                        else:
                            print(response.get('status_msg'))



    def interactive(self):
        if self.auth():
            self.file_check()
            while True:
                user_input = input(self.terminal_dis).strip()
                if not user_input:continue
                cmd_list = user_input.split()
                if hasattr(self,'_%s'%cmd_list[0]):
                    func = getattr(self,'_%s'%cmd_list[0])
                    func(cmd_list[1:])


    def send_msg(self,acttion_type,*args,**kwargs):
        '''将消息发送到远程'''
        my_data = {
            'action_type':acttion_type

        }
        my_data.update(kwargs)
        hear_json = json.dumps(my_data)
        hear_bytes = hear_json.encode('utf-8')
        self.sock.send(struct.pack('i',len(hear_bytes)))
        self.sock.send(hear_bytes)

    def parameter_check(self, args, min_args=None, max_args=None, exact_args=None):
        """参数个数合法性检查"""
        if min_args:
            if len(args) < min_args:
                print("must provide at least %s parameters but %s received." %(min_args,len(args)))
                return False
        if max_args:
            if len(args) > max_args:
                print("need at most %s parameters but %s received." %(max_args,len(args)))
                return False
        if exact_args:
            if len(args) != exact_args:
                print("need exactly %s parameters but %s received." % (exact_args, len(args)))
                return False
        return True

    def progress_bar(self,total_size,current_percent=0,last_percent=0):
        '''进度条'''
        while True:
            received_size = yield current_percent
            current_percent = int(received_size /total_size * 100)
            if current_percent > last_percent:
                print('>' * int(current_percent / 2) + '{percnet}%'.format(percnet=current_percent),end='
',
                      flush=True)
                last_percent = current_percent

    def _get(self,cmd_argv):
        '''下载文件'''
        if self.parameter_check(cmd_argv,min_args=1):
            filename = cmd_argv[0]
            self.send_msg(acttion_type='get',filename=filename)
            response = self.get_response()
            if response.get('status_code') == 301:
                file_size = response.get('file_size')
                recv_size = 0
                progress_gen = self.progress_bar(file_size)
                progress_gen.__next__()

                file_abs_path = os.path.join(self.current_dir,filename)
                self.shelve_obj[file_abs_path] = [file_size,'%s.download'%filename]

                with open('%s.download'%filename,'wb') as f:
                    while recv_size < file_size:
                        res = self.sock.recv(1024)
                        recv_size += len(res)
                        f.write(res)
                        progress_gen.send(recv_size)
                    else:
                        print('
')
                        print('下载成功！')
                        del self.shelve_obj[file_abs_path]
                os.rename('%s.download'%filename,filename)
            else:
                print(response.get('status_msg'))




    def _put(self,cmd_argv):
        '''上传文件'''
        if self.parameter_check(cmd_argv,exact_args=1):
            filename = cmd_argv[0]
            if os.path.isfile(filename):
                file_size = os.path.getsize(filename)
                self.send_msg(acttion_type='put',file_size=file_size,filename=filename)
                print('准备发送')
                send_size = 0
                progress_gen = self.progress_bar(file_size)
                progress_gen.__next__()
                with open(filename,'rb') as f:
                    for line in f:
                        self.sock.send(line)
                        send_size += len(line)
                        progress_gen.send(send_size)

                    else:
                        print('
')
                        print('上传成功！')


    def _ls(self,cmd_argv):
        '''查看'''
        self.send_msg(acttion_type='ls')
        response = self.get_response()
        if response.get('status_code') == 302:
            cmd_size = response.get('cmd_size')
            recv_size = 0
            cmd_result = b''
            while recv_size < cmd_size:
                res = self.sock.recv(1024)
                cmd_result += res
                recv_size += len(res)
            else:
                print(cmd_result.decode('gbk'))


    def _mkdir(self,cmd_argv):
        '''创建文件夹'''
        if self.parameter_check(cmd_argv,exact_args=1):
            inp = cmd_argv[0]
            self.send_msg(acttion_type='mkdir',inp=inp)
            response = self.get_response()
            if response.get('status_code') == 404:
                print(response.get('status_msg'))
            elif response.get('status_code') == 406:
                print(response.get('status_msg'))

    def _del(self,cmd_argv):
        '''删除文件'''
        if self.parameter_check(cmd_argv,exact_args=1):
            del_name = cmd_argv[0]
            self.send_msg(acttion_type='del',del_name=del_name)
            response = self.get_response()
            if response.get('status_code') == 405:
                print(response.get('status_msg'))
            elif response.get('status_code') == 407:
                print(response.get('status_msg'))

    def _rm(self,cmd_argv):
        '''删除文件夹'''
        if self.parameter_check(cmd_argv,exact_args=1):
            rm_name = cmd_argv[0]
            self.send_msg(acttion_type='rm',rm_name=rm_name)
            response = self.get_response()
            if response.get('status_code') == 500:
                print(response.get('status_msg'))
            elif response.get('status_code') == 501:
                print(response.get('status_msg'))

    def _cd(self,cmd_argv):
        '''切换目录'''
        if self.parameter_check(cmd_argv,exact_args=1):
            target_dir = cmd_argv[0]
            self.send_msg(acttion_type='cd',target_dir=target_dir)
            response = self.get_response()
            if response.get('status_code') == 350:
                self.terminal_dis = '[[%s]>>\%s]'% (self.username,response.get('current_dir'))
                self.current_dir = response.get('current_dir')





if __name__ == '__main__':
    obj = Myclient()
    obj.interactive()

 mythread.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*-
from threading import Thread
import queue

class Mythread:
    '''自定义线程池，实现并发'''
    def __init__(self,maxsize):
        self.maxsize = maxsize
        self.q = queue.Queue(maxsize) #  初始一个队列对象
        for i in range(maxsize):
            self.q.put(Thread) # 在队列对象中先存放线程对象

    def get_queue(self):
        '''从队列中取出线程对象'''
        return self.q.get()

    def put_queue(self):
        '''往队列存放线程对象'''
        return self.q.put(Thread)