爬虫——生产者消费者

结构

生产者生成网址并放入队列

多个消费者从队列中取出网址

from queue import Queue
import time, threading, requests

url_base = 'http://www.qiushibaike.com/8hr/page/{}/'
header = {}

def load_data():
    return [url_base.format(i) for i in [1, 3, 6, 7]]

#生产者
def produce(q):
    index = 0
    data = load_data()
    while True:
        if index < len(data):
            q.put(data[index])
            index += 1

#消费者
def consume(q):
    while True:
        download_url = q.get()
        # requests.get(download_url,headers=header)
        print('thread is {} content is {}'.format(threading.current_thread(), download_url))

def main():
    q = Queue(4)
    p1 = threading.Thread(target=produce, args=[q])
    c1 = threading.Thread(target=consume, args=[q])
    c2 = threading.Thread(target=consume, args=[q])
    p1.start()
    c1.start()
    c2.start()

if __name__ == '__main__':
    main()

类

爬虫类需要继承多线程类

初始化方法需要继承父类初始化方法

创建对象，直接start就会调用类中run方法

# class ConsumeSpider(threading.Thread):
#     def __init__(self):
#         super().__init__()
#         pass
#
#     def run(self):
#         pass
#
# c3 = ConsumeSpider()
# c3.start()

协程

协程(coroutine)：轻量级的线程，不存在上下文切换，能在多个任务之间调度的多任务方式，可以使用yield实现

import time, threading

def task_1():
    while True:
        print('-----1-----', threading.current_thread())
        time.sleep(1)
        yield


def task_2():
    while True:
        print('-----2-----', threading.current_thread())
        time.sleep(1)
        yield


def main():
    t1 = task_1()
    t2 = task_2()
    while True:
        next(t1)
        next(t2)


if __name__ == '__main__':
    main()
 

请使用手机"扫一扫"x