项目练习：电影列表爬虫

# -*- coding:utf-8 -*-
# Author:Sure Feng

import requests
import json


class DoubanFileSpider(object):
    def __init__(self):
        self.start_tempt_url = [
            {
            "url_tempt": "https://movie.douban.com/j/search_subjects?type=movie&tag=%E5%8D%8E%E8%AF%AD&sort=recommend&page_limit=20&page_start={}",
            "country": "cn"
            }, {
            "url_tempt": "https://movie.douban.com/j/search_subjects?type=movie&tag=%E6%97%A5%E6%9C%AC&sort=recommend&page_limit=20&page_start={}",
            "country": "janpan"
            }, {
            "url_tempt": "https://movie.douban.com/j/search_subjects?type=movie&tag=%E6%AC%A7%E7%BE%8E&sort=recommend&page_limit=20&page_start={}",
            "country": "usa"
            }
        ]
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Mobile Safari/537.36"}

    def parse_url(self, url):
        """发送请求，获取响应"""
        respond = requests.get(url, headers=self.headers)
        return respond.content.decode()

    def save_data(self, list_str, country):
        """保存数据"""
        with open("douban.txt", "a", encoding="utf-8") as f:
            for info_str in list_str:
                # print(info_str)
                info_str["country"] = country
                f.write(json.dumps(info_str, ensure_ascii=False))
                f.write("
")

    def get_content(self, json_str):
        """提取数据"""
        dict_ret = json.loads(json_str)
        content_list = dict_ret["subjects"]
        return content_list

    def run(self):
        """实现主要逻辑"""
        for url_tempt in self.start_tempt_url:
            num = 0
            country = url_tempt["country"]
            while True:
                # start_url
                start_url = url_tempt["url_tempt"].format(num)
                print(start_url)
                # 发送请求，获取响应
                json_str = self.parse_url(start_url)
                # 提取数据
                content_list = self.get_content(json_str)
                # 保存
                self.save_data(content_list, country)
                # 构造下一页的URL地址，重复步骤
                if len(content_list) < 20:
                    break
                num += 20


if __name__ == "__main__":
    douban_spider = DoubanFileSpider()
    douban_spider.run()