手动爬虫之报头及代理封装类（python3）

　　本人刚刚学习爬虫，见每次都需要添加报头比较繁琐，故将该过程封装为Url_ProxyHelper类，代码如下

import urllib.request as ur

class Url_ProxyHelper:
    def __init__(self, url, proxy_add, savepath=None):
        self.url = url
        self.proxy_add = proxy_add
        self.req = None
        self.proxy = None
        self.opener = None
        self.info = None
        self.save_path = savepath

    # 报头代理设置
    def set_UrlAndProxy(self):
        # 添加报头
        self.req = ur.Request(self.url)
        self.req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0')

        # 设置代理服务器
        self.proxy = ur.ProxyHandler({'http': self.proxy_add})
        self.opener = ur.build_opener(self.proxy, ur.HTTPHandler)
        ur.install_opener(self.opener)
        return self.req

    # 数据存档
    def save_InFile(self):
        self.req = self.set_UrlAndProxy()
        self.info = ur.urlopen(self.req).read()
        open(self.save_path, 'wb').write(self.info)

    # 数据返回
    def feedbak_info(self):
        self.req = self.set_UrlAndProxy()
        self.info = ur.urlopen(self.req).read().decode('utf-8')  # decode()用来解码，特别是中文
        return str(self.info)

测试：

from Url_ProxyHelper import Url_ProxyHelper

url = "https://www.baidu.com/"
save_path = "E:/workspace/PyCharm/codeSpace/books/python_web_crawler_book/chapter6/demo4/test_url_proxy.html"
proxy = "175.155.24.20:808"

# 调用Url_ProxyHelper类
#uph = Url_ProxyHelper(url, proxy, savepath=save_path)
uph = Url_ProxyHelper(url, proxy)

info = uph.feedbak_info()

print(info)