29、煲剧狂人

　　练习介绍

要求：

请使用多协程和队列，爬取时光网电视剧TOP100的数据（剧名、导演、主演和简介），并用csv模块将数据存储下来。

 

时光网TOP100链接：http://www.mtime.com/top/tv/top100/

 

目的：

1.练习掌握gevent的用法

2.练习掌握queue的用法

 

from gevent import monkey
monkey.patch_all()

from bs4 import BeautifulSoup
import gevent,requests,csv
from gevent.queue import Queue

url_list = ['http://www.mtime.com/top/tv/top100/']
for i in range(2,11):
    url_list.append('http://www.mtime.com/top/tv/top100/index-{}.html'.format(i))

work = Queue()

for url in url_list:
    work.put_nowait(url)

def pachong():
    while not work.empty():
        url = work.get_nowait()
        res = requests.get(url)
        items = BeautifulSoup(res.text,'html.parser').find_all('div',class_='mov_con')
        for item in items:
            title = item.find('h2').text.strip()
            director = 'null'
            actor    = 'null'
            remarks  = 'null'
            tag_ps = item.find_all('p')
            for tag_p in tag_ps:
                if tag_p.text[:2] == '导演':
                    director = tag_p.text[3:].strip()
                elif tag_p.text[:2] == '主演':
                    actor = tag_p.text[3:].strip().replace('	','')
                elif tag_p['class']:
                    remarks = tag_p.text.strip()
            with open('top100.csv','a',newline='',encoding='utf-8-sig') as csv_file:
                writer = csv.writer(csv_file)
                writer.writerow([title,director,actor,remarks])

task_list = []

for x in range(3):
    task = gevent.spawn(pachong)
    task_list.append(task)

with open('top100.csv','w',newline='',encoding='utf-8-sig') as csv_file:
    writer = csv.writer(csv_file)
    writer.writerow(['电视剧名','导演','主演','简介'])

gevent.joinall(task_list)

老师的代码

from gevent import monkey
monkey.patch_all()
import gevent,requests,bs4,csv
from gevent.queue import Queue

work = Queue()

url_1 = 'http://www.mtime.com/top/tv/top100/'
work.put_nowait(url_1)

url_2 = 'http://www.mtime.com/top/tv/top100/index-{page}.html'
for x in range(1,11):
    real_url = url_2.format(page=x)
    work.put_nowait(real_url)

def crawler():
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}
    while not work.empty():
        url = work.get_nowait()
        res = requests.get(url,headers=headers)
        bs_res = bs4.BeautifulSoup(res.text,'html.parser')
        datas = bs_res.find_all('div',class_="mov_con")
        for data in datas:
            TV_title = data.find('a').text
            data = data.find_all('p')
            TV_data =''
            for i in data:
                TV_data =TV_data + ''+ i.text
            writer.writerow([TV_title,TV_data])
            print([TV_title,TV_data])

csv_file = open('timetop.csv','w',newline='',encoding='utf-8-sig')
writer = csv.writer(csv_file)

task_list = []
for x in range(3):
    task = gevent.spawn(crawler)
    task_list.append(task)
gevent.joinall(task_list)