python爬虫优化和错误日志分析

发现问题

在爬虫下载过程中，执行一段时间后都会异常终止，下次必须kill掉进程重新运行 ，看能否优化并减少手动操作

错误日志分析

收集了nohup.out文件，发现主要错误是的数组下标越界，推测可能的问题为：
1）网络不稳定，http请求不通。
2）网络请求成功，但是html表单解析失败。
3）登录的cookie过期

优化思路

在所有有网络请求的地方，都加上了返回码是不是200的判断，然后html表单解析的地方加上数组长度判断，异常处理等

源码如下

import socket
import time
import os
from datetime import datetime
import re
import yaml
import requests
from bs4 import BeautifulSoup

# 设置超时时间为10s
socket.setdefaulttimeout(10)
s = requests.Session()


# 登录
def login():
    url = host_url + "j_spring_security_check"

    data = {
        "username": bzh_host_usr,
        "password": bzh_host_pwd
    }

    try:
        response = s.post(url, data=data, headers=headers)
        if response.status_code == 200:
            cookie = response.cookies.get_dict()
            print("login success")
            return cookie
    except Exception as e:
        print("login fail：", e)


# 页码
def get_pages():
    try:
        response = s.get(noticeListUrl, data=paramsNotice, headers=headers, cookies=cookie)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, "html.parser")
            pageCount = int(soup.find('span', id='PP_countPage').get_text())
            pageCount = pageCount if pageCount > 1 else 1
            return pageCount
    except Exception as e:
        print("get page_count fail：", e)


# 文档ids
def get_ids(pageCount):
    ids = []
    for p in range(int(pageCount)):
        paramsNotice['pageIndex'] = p + 1

        try:
            response = s.get(noticeListUrl, data=paramsNotice, headers=headers, cookies=cookie)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, "html.parser")
                trs = soup.find("table", class_='ObjectListPage').tbody.find_all("tr")
                regex = re.compile(r"noticeId=(d+)")

                for tr in trs:
                    if (tr.text.find("标准化文档更新") > 0):
                        id = regex.findall(str(tr))[0]
                        ids.append(id)
                        print("bzh id:" + id)

                        last_update = tr.find_all("td")[1].get_text().strip()
                        date_format = time.strftime("%Y%m%d", time.strptime(last_update, "%Y-%m-%d %H:%M:%S"))
                        file_name = "标准化文档-" + date_format + ".rar"

                        crawlFile(id, file_name)

        except Exception as e:
            print("get ids fail：", e)

    return ids


# 下载
def crawlFile(id, file_name):
    down_url = noticeURL + id
    metaFile = "./bzh/" + file_name

    response = s.get(down_url, headers=headers, cookies=cookie)
    content = response.headers.get('Content-Disposition')
    filename = content[content.find('=') + 1:]
    filename = filename.encode('iso-8859-1').decode('GBK')

    print("remote:" + filename)

    try:
        f = open(metaFile, 'wb')
        f.write(response.content)
        f.close()

        print(file_name + " first download success")
        exit(0)
    except Exception as e:
        print(file_name + " download fail", e)


if __name__ == "__main__":
    yaml_path = os.path.join('../', 'config.yaml')
    with open(yaml_path, 'r') as f:
        config = yaml.load(f, Loader=yaml.FullLoader)

    host_url = config['host_url']
    noticeListUrl = host_url + config['noticeListUrl']
    noticeDetailUrl = host_url + config['noticeDetailUrl']
    noticeURL = host_url + config['noticeURL']

    bzh_host_usr = config['bzh_host_usr']
    bzh_host_pwd = config['bzh_host_pwd']
    table_meta_bg_date = config['table_meta_bg_date']

    # header头信息
    headers = {
        "User-Agent": "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
        "Referer": host_url + "login.jsp"
    }

    paramsNotice = {
        "queryStartTime": table_meta_bg_date
    }

    task_begin = datetime.now()
    print("Crawler begin time:" + str(task_begin))

    cookie = login()
    if cookie == "":
        print("cookie is null")
        exit(0)

    pageCount = get_pages()

    pageCount = 2
    if pageCount < 1:
        print("page < 1")
        exit(0)

    ids = get_ids(pageCount)

    task_end = datetime.now()
    print("Crawler end time:" + str(task_end))

执行结果分析

优化后的爬虫运行正常，之前的异常已被捕获，输出在error日志里。
更新过的代码在线上环境跑了4天，收集了4天的错误日志，想从时间点上观察，看能否继续优化。

源码如下

import os
import matplotlib.pyplot as plt

if __name__ == "__main__":
    print("analyze error of bzh crawler")

    error_con = {}
    error_html = {}
    for i in range(0, 24):
        key = "0" + str(i) if i < 10 else str(i)
        error_con[key] = 0
        error_html[key] = 0

    error_file = os.popen('ls ' + "./input").read().split()
    for i in range(0, len(error_file)):
        input = open('./input/' + error_file[i], 'r')

        for line in input:
            lines = line.split()
            error_msg = line[line.find("-", 50) + 2:]
            hour = lines[2][0:2]

            if error_msg.find("get html failed") > -1:
                error_con[hour] += 1
            elif error_msg.find("parse detail html failed") > -1:
                error_html[hour] += 1 / 2

    # 折线图
    plt.title("Plot of Error Hour Analyze(20190507-20190510)")
    plt.xlabel("Hour")
    plt.ylabel("Error Count")

    plt.plot(error_con.keys(), error_con.values(), color="r", linestyle="-", marker="^", linewidth=1, label="connect")
    plt.plot(error_html.keys(), error_html.values(), color="b", linestyle="-", marker="s", linewidth=1,
             label="parse html")
    plt.legend(loc='upper left', bbox_to_anchor=(0.55, 0.95))

    plt.show()

输出折线图

• connect连接失败的次数明显多于parse解析失败，因为连接失败第一个页面就进不去了，也不存在后面的html解析
• 在连接正常的情况下，解析失败的次数占少数，4天的日志汇总，最多在1个小时里出现2次
• 2个折线图的走势基本一致，符合预期
• 折线图出现3个高峰，分别在凌晨4点，早上8点，晚上9点，推测远程服务器可能会定期重启，后期考虑是否加上爬虫时间过滤，晚上不执行来削峰
• 现在只有4天的日志，执行一段时间后收集长时间的日志，再观察是否和星期，天数，月份有关等