爬虫-豆瓣电影

豆瓣里可以搜到评分前250的电影。https://movie.douban.com/top250 ，可是分成了25页，我只能一页一页的看。想要一次看完250条，因此有了下面使用xpath的爬虫，把250条电影数据放进csv文件：

import requests
import lxml.html
import csv

doubanUrl = 'https://movie.douban.com/top250?start={}&filter=' #{}用于匹配参数

def getSource(url):
    '''
    获取网页源代码。
    :param url:
    :return: String
    '''
    content = requests.get(url)
    content.encoding = 'utf-8'          #万能编码
    #强制修改编码,防止Windows下出现乱码
    return content.content


def getEveryItem(source):
    '''
    获取每一个电影的相关信息。movie_dict字典用于保存电影的信息。
    :param source:
    :return: [movie1_dict, movie2_dict, movie3_dict,...]
    '''
    selector = lxml.html.document_fromstring(source)
    movieItemList= selector.xpath('//div[@class="info"]') #如果不从html开始都需要//  此处使用到了先抓大再抓小的技巧
    movieList = []


    for eachMoive in movieItemList:
        movieDict = {}
        title = eachMoive.xpath('div[@class="hd"]/a/span[@class="title"]/text()')
        print(title)
        otherTitle = eachMoive.xpath('div[@class="hd"]/a/span[@class="other"]/text()')

        link = eachMoive.xpath('div[@class="hd"]/a/@href')[0]
        print('看看多少a标签')
        print(eachMoive.xpath('div[@class="hd"]/a/@href')[:])



        directorAndActor = eachMoive.xpath('div[@class="bd"]/p[@class=""]/text()')
        star = eachMoive.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')[0]
        quote = eachMoive.xpath('div[@class="bd"]/p[@class="quote"]/span/text()')
        if quote:
            quote = quote[0]
        else:
            quote = ''

        movieDict['title'] = ''.join(title + otherTitle)
        movieDict['url'] = link
        #你可以试一试直接打印''.join(directorAndActor),看看他的格式是多么的混乱
        movieDict['directorAndActor'] = ''.join(directorAndActor).replace(' ', '').replace('
', '').replace('
', '') #可以同时多个替换
        movieDict['star'] = star
        movieDict['quote'] = quote

        movieList.append(movieDict) #把抓到的每个电影房进集合里
    return movieList

def writeData(movieList):
    with open('doubanMovie_example2.csv', 'w', encoding='UTF-8', newline='') as f:  #需要新建个doubanMovie_example2.csv文件
        writer = csv.DictWriter(f, fieldnames=['title', 'directorAndActor', 'star', 'quote', 'url'])
        writer.writeheader()
        for each in movieList:
            print(each)
            writer.writerow(each)

if __name__ == '__main__':
    movieList = []
    for i in range(10):
        pageLink = doubanUrl.format(i * 25) #给url格式化 每次拿25条 从第一页开始即i为0开始  ，世事无绝对还得看爬虫网址的规律。
        print(pageLink)
        source = getSource(pageLink)
        movieList += getEveryItem(source)
    print(movieList[:10]) #把十页数据即250条拿来看看
    movieList = sorted(movieList, key=lambda k: k['star'], reverse=True) #根据字典中的key=star这一项的value来排序,倒序。
    writeData(movieList)

显示结果：

1、控制台显示

2、csv文件，用excel打开，可能中文出现乱码，这时用记事本打开并且另存为utf-8文件，再打开就好了。