使用pyquery爬取豆瓣电影top250，存储在mongodb

直接上代码

from pyquery import PyQuery as pq
from requests import request
import re
import pymongo


class SpiderDouBan2:


    def __init__(self):
        client = pymongo.MongoClient(host='localhost', port=27017)
        db = client['spider_db']
        self.collection = db['douban_movie_top250_2']



    def get_doc(self, url):
        '''
        获取某一页的doc
        :param url: 地址
        :return:
        '''
        doc = pq(url)
        return doc


    def get_one_page(self, doc, order):
        '''
        获取某一页的内容
        :return:
        '''
        movie_names = [name.text() for name in doc('.title').items() if not re.search('/', name.text())]
        movie_actors = [re.sub('xa0|" "', '', actor.text()).split('
') for actor in doc('.info .bd')('p:first-child').items()]
        movie_rates = [rate.text() for rate in doc('.rating_num').items()]
        comment_nums = [comment_num.text() for comment_num in doc('.star')('span:last-child').items()]
        short_comments = [short_comment.text()[:-1] for short_comment in doc('.inq').items()]
        for index, name in enumerate(movie_names):
            print(f'正在爬取第{order + index + 1}条数据...')
            movie_info = {
                        'order': f'No.{order + index + 1}',
                        'movie_name': name,
                        'movie_actor': movie_actors[index][0].rstrip('/'),
                        'movie_type': movie_actors[index][1],
                        'movie_rate': f'{movie_rates[index]}分',
                        'comment_num': comment_nums[index],
                        'short_comment': short_comments[index]
                        }
            self.collection.insert_one(movie_info)



    def main(self, url, order):
        '''
        主程序
        :return:
        '''
        doc = self.get_doc(url)
        self.get_one_page(doc, order)


if __name__ == '__main__':
    for offset in range(0, 250, 25):
        order = offset
        url = f'https://movie.douban.com/top250?start={str(offset)}'
        SpiderDouBan2().main(url, order)

运行结果