模拟登陆/代理服务器/cookie/

代理：代理服务器

代理：代理服务器
    - 快代理
    - 西祠代理
    - goubanjia
匿名度：
    透明：对方服务器知道你使用了代理ip也知道你的真实ip
    匿名：知道你使用了代理ip但是不知道你的真实ip
    高匿：什么都不知道
类型：
    http：只可以发起http请求
    https：只可以发起https的请求

cookie的应用和处理

- cookie:服务器端记录客户端的相关状态
- 处理cookie的方式：
    - 手动处理：不建议
    - 自动处理：回话对象Session，该对象可以像requests模块一样进行网络请求的发送（get，post）。session进行的请求发送可以自动携带和处理cookie。

#基于cookie的案例分析：https://xueqiu.com/
#1.从首页中获取详情页的url
    #发现：首页中的新闻数据是动态加载出来（ajax） json数据中taget对应的value值就是详情页的url
import requests
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}
#自动获取cookie，cookie就会自动存储到session中
session = requests.Session()
session.get('https://xueqiu.com/',headers=headers)

#捕获ajax数据包获取的url
url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=10&category=-1'
#携带cookie进行的请求发送
dic_json = session.get(url=url,headers=headers).json()
print(dic_json)
#从响应数据中获取详情页的url
# for dic in dic_json['list']:
# #     print(dic)
#     d = dic['data']
#     detail_url = 'https://xueqiu.com'+d['target']
#     print(detail_url)

如何提升requests模块爬取数据的效率

多进程或者多线程（不建议）

线程池或者进程池（适当使用）

单线程+异步协程（推荐）

import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool
import random
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}


def request_video(url):
    return requests.get(url=url,headers=headers).content

def saveVideo(data):
    name = str(random.randint(0,9999))+'.mp4'
    with open(name,'wb') as fp:
        fp.write(data)
        print(name,'下载存储成功！！！')

url = 'https://www.pearvideo.com/category_1'
page_text = requests.get(url=url,headers=headers).text

tree = etree.HTML(page_text)
li_list = tree.xpath('//ul[@id="listvideoListUl"]/li')
#实例化一个线程池对象
pool = Pool(4)
video_url_list = [] #所有的视频连接
for li in li_list:
    detail_url = 'https://www.pearvideo.com/'+li.xpath('./div/a/@href')[0]
    detail_page_text = requests.get(url=detail_url,headers=headers).text
    ex = 'srcUrl="(.*?)",vdoUrl='
    video_url = re.findall(ex,detail_page_text,re.S)[0]
    video_url_list.append(video_url)
#异步的获取4个视频的二进制数据
video_data_list = pool.map(request_video,video_url_list)

#进行视频的持久化存储
pool.map(saveVideo,video_data_list)

云打码平台注册和使用

相关的门户网站在进行登录的时候，如果用户连续登录的次数超过3次或者5次的时候，就会在登录页中动态生成验证码。通过验证码达到分流和反爬的效果

云打码网址,

云打码平台处理验证码的实现流程

- 1.对携带验证码的页面数据进行抓取
- 2.可以将页面数据中验证码进行解析，验证码图片下载到本地
- 3.可以将验证码图片提交给三方平台进行识别，返回验证码图片上的数据值
    - 云打码平台：
        - 1.在官网中进行注册（普通用户和开发者用户）
        - 2.登录开发者用户：
            - 1.实例代码的下载（开发文档-》调用实例及最新的DLL-》PythonHTTP实例下载）
            - 2.创建一个软件：我的软件-》添加新的软件
        -3.使用示例代码中的源码文件中的代码进行修改，让其识别验证码图片中的数据值

代码展示

import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

######################################################################

# 用户名,云打码品台普通用户的用户名
username    = 'username'

# 密码,云打码平台普通用户的密码
password    = 'password'                            

# 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
appid       = 1                                     

# 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
appkey      = '22cc5376925e9387a23cf797cb9ba745'    

# 图片文件
filename    = 'getimage.jpg'                        

# 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
codetype    = 1004

# 超时时间，秒
timeout     = 60                                    

# 检查
if (username == 'username'):
    print('请设置好相关参数再测试')
else:
    # 初始化
    yundama = YDMHttp(username, password, appid, appkey)

    # 登陆云打码
    uid = yundama.login();
    print('uid: %s' % uid)

    # 查询余额
    balance = yundama.balance();
    print('balance: %s' % balance)

    # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
    cid, result = yundama.decode(filename, codetype, timeout);
    print('cid: %s, result: %s' % (cid, result))

######################################################################

实例:人人网携带验证码的模拟登陆

from lxml import etree
from urllib import request
import http.client, mimetypes, urllib, json, time, requests

######################################################################

class YDMHttp:

    apiurl = 'http://api.yundama.com/api.php'
    username = ''
    password = ''
    appid = ''
    appkey = ''

    def __init__(self, username, password, appid, appkey):
        self.username = username  
        self.password = password
        self.appid = str(appid)
        self.appkey = appkey

    def request(self, fields, files=[]):
        response = self.post_url(self.apiurl, fields, files)
        response = json.loads(response)
        return response
    
    def balance(self):
        data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['balance']
        else:
            return -9001
    
    def login(self):
        data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey}
        response = self.request(data)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['uid']
        else:
            return -9001

    def upload(self, filename, codetype, timeout):
        data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)}
        file = {'file': filename}
        response = self.request(data, file)
        if (response):
            if (response['ret'] and response['ret'] < 0):
                return response['ret']
            else:
                return response['cid']
        else:
            return -9001

    def result(self, cid):
        data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)}
        response = self.request(data)
        return response and response['text'] or ''

    def decode(self, filename, codetype, timeout):
        cid = self.upload(filename, codetype, timeout)
        if (cid > 0):
            for i in range(0, timeout):
                result = self.result(cid)
                if (result != ''):
                    return cid, result
                else:
                    time.sleep(1)
            return -3003, ''
        else:
            return cid, ''

    def report(self, cid):
        data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'}
        response = self.request(data)
        if (response):
            return response['ret']
        else:
            return -9001

    def post_url(self, url, fields, files=[]):
        for key in files:
            files[key] = open(files[key], 'rb');
        res = requests.post(url, files=files, data=fields)
        return res.text

######################################################################

#封装识别验证码图片的函数
def getCodeText(codeType,filePath):
    result = None
    # 云打码品台普通用户的用户名
    username    = '1213242044'
    # 云打码品台普通用户的密码
    password    = 'wanglan000'                            
    # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appid       = 7569                                  
    # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
    appkey      = '7b57f886d213ec9b43521422012d8a87'    
    # 图片文件
    filename    = filePath                     
    # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
    codetype    = codeType
    # 超时时间，秒
    timeout     = 120                                   
    # 检查
    if (username == 'username'):
        print('请设置好相关参数再测试')
    else:
        # 初始化
        yundama = YDMHttp(username, password, appid, appkey)
        # 登陆云打码
        uid = yundama.login();
        print('uid: %s' % uid)
        # 查询余额
        balance = yundama.balance();
        print('balance: %s' % balance)
        # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
        cid, result = yundama.decode(filename, codetype, timeout);
        print('cid: %s, result: %s' % (cid, result))
    return result

url = "http://www.renren.com/SysHome.do"
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}
page_text = requests.get(url=url,headers=headers,proxies={'http':'183.129.244.21:14297'}).text #proxies:代理

#解析出验证码图片的地址
tree = etree.HTML(page_text)
code_img_url = tree.xpath('//*[@id="verifyPic_login"]/@src')[0]
request.urlretrieve(url=code_img_url,filename='./code.jpg')

#使用打码平台识别验证码
code_text = getCodeText(2004,'./code.jpg')
print(code_text)

# 模拟登陆
login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019401928941' #注意url连接有时间戳
data = {
    "email": "1213242044",
    "icode": code_text,
    "origURL": "http://www.renren.com/home",
    "domain": "renren.com",
    "key_id": "1",
    "captcha_type": "web_login",
    "password": "17108719a8907d7a716631afc519561080a50ea31d92f64ae5f9ccd64b41e4b1", #会随着时间戳变化而变化
    "rkey": "465733ef9b05bd790ec81e9347025808", #会随着时间戳变化而变化
    "f": "http%3A%2F%2Fwww.renren.com%2F289676607",
}
#创建一个回话对象
session = requests.Session()
#产生cookie
response = session.post(url=login_url,headers=headers,data=data,proxies={'http':'119.180.135.210:8060'})
# print(response.status_code)  #查看登陆状态

# page_text = response.text
# print(page_text)

#该次请求发送必须携带cookie
detail_url = 'http://www.renren.com/289676607/profile'
detail_page_text = session.get(url=detail_url,headers=headers).text