爬虫实战—模拟登陆oschina

1、模拟登陆oschina（新浪）

　　一般登录后，用户就可以一段时间内可以使用该用户身份操作，不需要频繁登录。这背后往往使用了Cookie技术

　　登录后，用户获得一个cookie 值，这个值在浏览器当前会话中保存，只要不过期甚至可以保存很久

　　用户每次想服务器提交请求时，将这些cookie提交到服务器，服务器经过分析cookie中的信息，以确认用户身份，确认是信任的用户身份，就可以继续使用网站功能。

　　Cookie：网景公司发明，cookie 一般是一个键值对name=value ,但还可以包括 expire 过期，path路径，domain域， secure安全等信息。

　　　　

　　清空 oschina.net 的 所有cookie 重新登录，勾选“记住密码”

　　　　

　　登录后的请求头如下：　　　

GET /?nocache=1544444524642 HTTP/1.1
Host: www.oschina.net
Connection: keep-alive
Cache-Control: max-age=0
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3486.0 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Referer: https://www.oschina.net/home/login?goto_page=https%3A%2F%2Fwww.oschina.net%2F
Accept-Encoding: gzip, deflate, br
Accept-Language: zh-CN,zh;q=0.9,en;q=0.8
Cookie: _user_behavior_=d2104a4f-2484-4f85-8a31-4fe2a86accb8; aliyungf_tc=AQAAAAR/MWXo0QAAV8CVPSF2shLDVU11; Hm_lvt_a411c4d1664dd70048ee98afe7b28f0b=1544444408; _reg_key_=foI49279hton2EYg1ZJz; socialauth_id=n6SsxSVbY6yycMzklFO7; oscid=ZV2oveUqo28xv80qumQtfRqukWzpKq2brNqjn0Y0a5kFTeUQUUbcPj2dwLIiVt%2FuqEFRQShwYl7DjeTX5ZGViddJVodYy0RwW38eexYn%2FPq9afSRNy7SJarEKkqVYfw%2BdNYj1bbHQEhDiqhDeFBZbsf7ouMp1Msoa4cH6mU1ZtM%3D; Hm_lpvt_a411c4d1664dd70048ee98afe7b28f0b=1544444525

　　对比登录前后的cookie 值，发现登录后又oscid

　　那就 把这个登录后的HTTP 请求头放在代码中:

　　技巧： 使用postman 工具：

　　　　

　　　　

　　　　　　代码如下：（修改后）　　

import requests

url = "https://www.oschina.net"


headers = {
    'Host': "www.oschina.net",
    'Connection': "keep-alive",
    'Cache-Control': "max-age=0",
    'Upgrade-Insecure-Requests': "1",
    'User-Agent': "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3486.0 Safari/537.36",
    'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    'Referer': "https://www.oschina.net/home/login?goto_page=https%3A%2F%2Fwww.oschina.net%2F",
    'Accept-Encoding': "gzip, deflate, br",
    'Accept-Language': "zh-CN,zh;q=0.9,en;q=0.8",
    'Cookie': "_user_behavior_=d2104a4f-2484-4f85-8a31-4fe2a86accb8; aliyungf_tc=AQAAAAR/MWXo0QAAV8CVPSF2shLDVU11; Hm_lvt_a411c4d1664dd70048ee98afe7b28f0b=1544444408; _reg_key_=foI49279hton2EYg1ZJz; socialauth_id=n6SsxSVbY6yycMzklFO7; oscid=ZV2oveUqo28xv80qumQtfRqukWzpKq2brNqjn0Y0a5kFTeUQUUbcPj2dwLIiVt%2FuqEFRQShwYl7DjeTX5ZGViddJVodYy0RwW38eexYn%2FPq9afSRNy7SJarEKkqVYfw%2BdNYj1bbHQEhDiqhDeFBZbsf7ouMp1Msoa4cH6mU1ZtM%3D; Hm_lpvt_a411c4d1664dd70048ee98afe7b28f0b=1544444525",
    'cache-control': "no-cache",
    'Postman-Token': "7d3714a6-c3d7-45ef-9b14-815ffb022535"
    }

response = requests.request("GET", url, headers=headers)

with response:
    with open('f://text.html','w',encoding='utf-8') as f:
        text = response.text
        f.write(text)
        print(text)
        print(response.status_code,'==========')

　　　　　　输出文件：

　　

2、多线程爬取博客园

　　博客园的新闻分页地址：https://news.cnblogs.com/n/page/10/， 多线程成批 爬取新闻标题和连接

　　https://news.cnblogs.com/n/page/2/  这个url 中变化的是最后的数字一直在变，它是页码　

import requests
from  concurrent.futures import  ThreadPoolExecutor
from queue import Queue
from  bs4 import BeautifulSoup
import threading
import time
import logging

FORMAT = "%(asctime)s %(threadName)s %(thread)s %(message)s"
logging.basicConfig(format=FORMAT, level=logging.INFO)

BASE_URL = "https://news.cnblogs.com"
NEW_PAGE = '/n/page/'

headers = {
    'User-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.4.3000'
}

# 使用池，以后可以使用第三方消息队列完成
urls = Queue()  # url 的队列
htmls = Queue() # 响应数据队列
outputs = Queue() # 结果输出队列

# 创建博客园的新闻urls，每页30条新闻
def create_url(start, end, step=1):
    for i in range(start, end + 1, step):
        url = '{}{}{}/'.format(BASE_URL,NEW_PAGE, i)
        print(url)
        urls.put(url)
    print('urls创建完毕')

event = threading.Event()

# 爬取页面线程函数
def crawler():
    while not event.is_set():
        try:
            url = urls.get(True, 1)
            with requests.request('GET', url , headers=headers) as response:
                html = response.text
                htmls.put(html)
        except:
            pass
# 解析线程函数
def parse():
    while not event.is_set():
        try:
            html = htmls.get(True, 1)
            soup =BeautifulSoup(html, 'lxml')
            titles = soup.select('h2.news_entry a')
            for title in titles:
                # <a href='/n/60287/' target='_blank'> 特斯拉</a>
                val = (BASE_URL + title.attrs['href'], title.text)
                outputs.put(val)
                print(val)
        except:
            pass
# 持久化线程函数
def persist(path):
    with open(path, 'a+', encoding='utf-8') as f:
        while not event.is_set():
            try:
                url, text = outputs.get(True, 1)
                print(url, text)
                f.write('{}x01{}'.format(url, text))
                f.flush()
            except:
                pass

# 线程池
executor = ThreadPoolExecutor(10)

executor.submit(create_url, 1, 10) # 模拟url收集,结束后，线程权让出
executor.submit(persist, 'f;/new.txt')

# 爬取页面并分析
for i in range(5):
    executor.submit(crawler)
for i in range(4):
    executor.submit(parse)


while True:
    cmd = input('>>>>>>')
    if cmd.strip() == 'quit':
        event.set()
        time.sleep(4)
        break
    print(threading.enumerate())

 　　　　解析内容是一个比较耗时的过程，不适合放在crawler中同步处理，同样适用队列解耦

　　　　 html 分析函数，parse ，分析完成后，需要将结果持久化， 不要在parse中直接持久化，放入队列中，统一持久化

　　　　这样一个实用的并行的爬虫就基本完成了

　　　　可以很方便的扩展成多进程版本

3、进阶（使用消息队列）

 　　将队列换成第三方服务，本次采用较为常用的RabbitMQ

　　搭建RabbitMQ服务

 　　队列工作模式选择：

　　　　　以爬虫程序的htmls 队列为例，这个队列有很多个生产者（爬取函数 ）写入，有多个消费者（解析函数）读取每一个消息只需要一个消费者使用，所以采用 RabbitMQ 的工作队列模式。

　　队列中如何分发呢：

　　　　　　其实说到底是路由，RabbitMQ的队列和工作队列，其实都是路由模式，只不过使用了缺省交换机

　　队列是否断开删除：　

 　　　　　　每一数据都要处理，不能因为某一端断开，然后队列就删除了，造成数据丢失。

　　测试代码：

　　　　　send.py　　　

import pika
import time

exchange = 'news'
queue = 'urls'

params = pika.URLParameters('amqp://rab:123456@192.168.112.111:5672/test')

connection = pika.BlockingConnection(params)
channel = connection.channel()

# 生成一个交换机
channel.exchange_declare(
    exchange=exchange,
    exchange_type='direct'
)

channel.queue_declare(queue, exclusive=False)  # 生成队列
# 绑定队列到交换机， 没有指定routing_key ，将使用队列名
channel.queue_bind(queue, exchange)

with connection:
    for i in range(10):
        msg = 'data{:02}'.format(i)  # 让消息带上routing_key 便于观察
        pub = channel.basic_publish(
            exchange=exchange,
            routing_key=queue,  # 指定routing_key ，没有指定，就使用队列名称匹配
            body=msg  # 消息
        )
        print(msg, '==================')

    print('===== send ok ===========')

 　　　　　　receive.py

import pika
import time

exchange = 'news'
queue = 'urls'

params = pika.URLParameters('amqp://rab:123456@192.168.112.111:5672/test')

connection = pika.BlockingConnection(params)
channel = connection.channel()

# 生成一个交换机
channel.exchange_declare(
    exchange=exchange,
    exchange_type='direct'
)

channel.queue_declare(queue, exclusive=False)  # 生成队列
# 绑定队列到交换机， 没有指定routing_key ，将使用队列名
channel.queue_bind(queue, exchange)

time.sleep(2)
with  connection:
    msg = channel.basic_get(queue, True) # 从指定的队列获取一个消息
    method , props, body = msg
    if body:
        print(body)
    else:
        print('empty')

 　　　　　　重复获取消息：

import pika
import time

exchange = 'news'
queue = 'urls'

params = pika.URLParameters('amqp://rab:123456@192.168.112.111:5672/test')

connection = pika.BlockingConnection(params)
channel = connection.channel()

# 生成一个交换机
channel.exchange_declare(
    exchange=exchange,
    exchange_type='direct'
)

channel.queue_declare(queue, exclusive=False)  # 生成队列
# 绑定队列到交换机， 没有指定routing_key ，将使用队列名
channel.queue_bind(queue, exchange)


def callback(channel, method, properties, body):
    print(body)


tag = None
def cancel(tag):
    print(tag)
    channel.basic_cancel(tag) # 取消basic_consume

import threading

time.sleep(10)

def start():
    with  connection:
        tag = channel.basic_consume(
            callback,
            queue,
            True
        ) # 从指定队列获取一个消息回调，tag获取不到，阻塞,获取到，表示结束了
        threading.Timer(10, cancel, args=(tag,)).start()
        channel.start_consuming() # 等待所有的basic_consume消费者消费完， 就结束

threading.Thread(target=start).start() # 由于channel.start_consuming() 会阻塞，再开启一个线程
print('======== end ===========')

 　　　　注：上面的多线程代码，写的不好，大量使用了全局变量，只是为了说明问题

重构消息队列：

import pika
import time
import threading

class MessageQueue:
    def __init__(self, host,port,user,password, vhost, exchange,queue):
        url = 'amqp：//{}:{}@{}:{}/{}'.format(
            user,password,host, port,vhost
        )
        params = pika.URLParameters(url)
        self.connection = pika.BlockingConnection(params)
        self.channel = self.connection.channel()
        self.exchange = self.channel.exchange_declare(exchange, 'direct')
        self.exchange_name = exchange
        self.channel.queue_declare(queue, exclusive=False) # 生成队列
        self.queue = queue # 队列名，当routing_key
        self.channel.queue_bind(queue, exchange)

    def __enter__(self):
        return self.channel

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connection.close()# 关闭连接s

# 生产者
class Producter(MessageQueue):
    def sendmsg(self, msg):
        self.channel.basic_publish(
            exchange=self.exchange_name,
            routing_key=self.queue,
            body = msg
        )

# 消费者
class Consumer(MessageQueue):
    def recvmsg(self):
        return  self.channel.basic_get(self.queue, True)[2] # body

重构爬虫代码：

import requests
from  concurrent.futures import  ThreadPoolExecutor
from queue import Queue
from  bs4 import BeautifulSoup
import threading
import time
import logging
import pika
import simplejson
from messagequeue import Producter, Consumer

FORMAT = "%(asctime)s %(threadName)s %(thread)s %(message)s"
logging.basicConfig(format=FORMAT, level=logging.INFO)

BASE_URL = "https://news.cnblogs.com"
NEW_PAGE = '/n/page/'

headers = {
    'User-agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.4.3000'
}

# 创建博客园的新闻urls，每页30条新闻
def create_url(start, end, step=1):
    try:
        p = Producter('192.168.112.111',5672,'rab','123456','test','news','urls')
        for i in range(start, end + 1, step):
            url = '{}{}{}/'.format(BASE_URL,NEW_PAGE, i)
            print(url)
            p.sendmsg(url)
        print('urls创建完毕')
    except Exception as e:
        print(e)

event = threading.Event()

# 爬取页面线程函数
def crawler():
    try:
        p = Producter('192.168.112.111', 5672, 'rab', '123456', 'test', 'news', 'htmls')
        c = Consumer('192.168.112.111', 5672, 'rab', '123456', 'test', 'news', 'urls')
        while not event.wait(1):
            try:
                # url = urls.get(True, 1)
                url = c.recvmsg()
                with requests.request('GET', url , headers=headers) as response:
                    html = response.text
                    p.sendmsg(html)
            except:
                raise
    except Exception as e:
        print(e)
# 解析线程函数
def parse():
    try:
        p = Producter('192.168.112.111', 5672, 'rab', '123456', 'test', 'news', 'outputs')
        c = Consumer('192.168.112.111', 5672, 'rab', '123456', 'test', 'news', 'htmls')
        while not event.wait(1):
            try:
                # html = htmls.get(True, 1)
                html = c.recvmsg()
                if html:
                    soup =BeautifulSoup(html, 'lxml')
                    titles = soup.select('h2.news_entry a')
                    for title in titles:
                        # <a href='/n/60287/' target='_blank'> 特斯拉</a>
                        # val = (BASE_URL + title.attrs['href'], title.text)
                        # outputs.put(val)
                        val = simplejson.dumps({
                            'title':title.text,
                            'url':BASE_URL + title.attrs['href']
                        })
                        p.sendmsg(val)
                        print(val)
            except:
                raise
    except Exception as e:
        print(e)
# 持久化线程函数
def persist(path):
    try:
        c = Consumer('192.168.112.111', 5672, 'rab', '123456', 'test', 'news', 'outputs')
        with open(path, 'a+', encoding='utf-8') as f:
            while not event.is_set():
                try:
                    # url, text = outputs.get(True, 1)
                    data = c.recvmsg()
                    print(data,'==========================================')
                    print(type(data))
                    if data :
                        d = simplejson.loads(data)
                        print(d,'------------------------------------------')
                        print(type(d))
                        # print(url, text)
                        f.write('{}x01{}'.format(d['url'],d['title']))
                        f.flush()
                except:
                    pass
    except Exception as e:
        print( e)
# 线程池
executor = ThreadPoolExecutor(10)

executor.submit(create_url, 1, 10) # 模拟url收集,结束后，线程权让出
executor.submit(persist, 'f;/new.txt')

# 爬取页面并分析
for i in range(5):
    executor.submit(crawler)
for i in range(4):
    executor.submit(parse)


while True:
    cmd = input('>>>>>>')
    if cmd.strip() == 'quit':
        event.set()
        time.sleep(4)
        break
    print(threading.enumerate())