python爬虫实战项目

python爬虫实战项目

1. LOL所有英雄皮肤下载

from fake_useragent import UserAgent
import requests, json, os

# 爬取网页所有英雄的皮肤图片
# https://lol.qq.com/data/info-heros.shtml


# 获取英雄id
def get_heroList():
    url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js'
    headers = {
        'User-Agent': UserAgent().chrome
    }
    try:
        response = requests.get(url, headers=headers)
        # print(response.text)
        # print(type(response.text))
        response = json.loads(response.text)
        # print(type(response))
        hero_ids = []
        for i in response['hero']:
            hero_ids.append(i['heroId'])
        # print(hero_ids)
        return hero_ids
    except:
        print('获取英雄id失败')
        return None


# 根据英雄id获取英雄皮肤名称和图片下载地址
def get_skinNames(id):
    url = 'https://game.gtimg.cn/images/lol/act/img/js/hero/{}.js'.format(id)
    headers = {
        'User-Agent': UserAgent().chrome
    }
    try:
        response = requests.get(url, headers=headers)
        response = json.loads(response.text)
        skinnames = []
        skin_urls = []
        for i in response['skins'][:-1]:
            if i['mainImg'] != '':
                skinnames.append(i['name'])
                skin_urls.append(i['mainImg'])
        # print(skinnames)
        return skinnames, skin_urls
    except:
        print('获取英雄皮肤名称失败')
        return None


# 根据名称,下载图片保存文件夹
def downloadImg(skinnames, skin_urls):
    headers = {
        'User-Agent': UserAgent().chrome
    }
    filename = skinnames[0]
    os.makedirs(filename, exist_ok=True)
    for skinname, skin_url in zip(skinnames, skin_urls):
        try:
            response = requests.get(skin_url, headers=headers)
        except:
            print(skinname + ' 下载失败')
            return
        with open(filename+'/'+skinname.replace('/', '_') + '.jpg', 'wb') as f:
            f.write(response.content)
    # print(filename + ' 下载完成')


if __name__ == '__main__':
    hero_ids = get_heroList()
    i = 1
    for id in hero_ids:
        skinnames, skin_urls = get_skinNames(id)
        # print(skinnames[0]+':'+str(len(skin_urls))+'张')
        downloadImg(skinnames, skin_urls)
        print('
下载进度:' + str(i) + '/' + str(len(hero_ids)), end='')
        i = i + 1

2. 音乐下载软件

import requests, json, re
from tkinter import Tk, Button, Entry, StringVar, Radiobutton, Frame
from tkinter import messagebox


# 说明:
# 爬取网站:https://music.zhuolin.wang/
# ajax异步请求
# 下载的歌曲在软件所在目录下


# 根据输入找到歌曲信息
def get_musicInfo(query, sourse):
    music_ids = []
    music_names = []
    music_singers = []
    url = 'https://music.zhuolin.wang/api.php?'
    data = {
        'types': 'search',
        'count': '5',
        'source': sourse,
        'pages': '1',
        'name': query
    }
    headers = {
        'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Content-Length': '37',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Cookie': '',
        'Host': 'music.zhuolin.wang',
        'Origin': 'https://music.zhuolin.wang',
        'Referer': 'https://music.zhuolin.wang/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest'
    }
    response = requests.post(url, headers=headers, data=data)
    response = json.loads(response.text)
    for i in response:
        music_ids.append(i['id'])
        music_names.append(i['name'])
        music_singers.append(i['artist'])

    print(music_ids)
    print(music_names)
    print(music_singers)
    # return music_ids, music_names, music_singers


# 根据id获取歌曲下载链接
def get_downloadUrl(music_id, name, singer, sourse):
    url = 'https://music.zhuolin.wang/api.php?'
    data = {
        'types': 'url',
        'id': music_id,
        'source': sourse
    }
    headers = {
        'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Content-Length': '37',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Cookie': '',
        'Host': 'music.zhuolin.wang',
        'Origin': 'https://music.zhuolin.wang',
        'Referer': 'https://music.zhuolin.wang/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest'
    }
    response = requests.post(url, data=data, headers=headers)
    print(response.text)
    downloadurl = re.search(r'http:(.+)",', response.text)
    if downloadurl != None:
        downloadurl = downloadurl.group().replace('\', '')
        downloadMusic(downloadurl, name, singer)
    else:
        messagebox.showinfo('抱歉', '该歌曲暂不提供下载,请您更换其他平台下载')


# 下载歌曲到本地
def downloadMusic(url, name, singer):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36'
    }
    try:
        response = requests.get(url, headers=headers)
        with open(name + '-' + singer + '.mp3', 'wb')as f:
            f.write(response.content)
        messagebox.showinfo('恭喜', name + '-' + singer + ' 下载完成')
    except:
        messagebox.showinfo('抱歉', name + ' 下载失败')


# 点击搜索执行
def search_music():
    query = entry.get()
    sourse = v.get()
    if query == '':
        messagebox.showinfo('提示', '请输入内容!')
        return False
    music_ids, music_names, music_singers = get_musicInfo(query, sourse)
    # 重新进行组件内容和按钮功能的设置
    for i in range(5):
        if i == 0:
            id1 = str(music_ids[i])
            name1 = str(music_names[i])
            singer1 = str(music_singers[i][0])
            value1.set(name1 + '   ' + singer1)
            entry1['textvariable'] = value1
            button1['command'] = lambda: download(id1, name1, singer1)
        if i == 1:
            id2 = str(music_ids[i])
            name2 = str(music_names[i])
            singer2 = str(music_singers[i][0])
            value2.set(name2 + '   ' + singer2)
            entry2['textvariable'] = value2
            button2['command'] = lambda: download(id2, name2, singer2)
        if i == 2:
            id3 = str(music_ids[i])
            name3 = str(music_names[i])
            singer3 = str(music_singers[i][0])
            value3.set(name3 + '   ' + singer3)
            entry3['textvariable'] = value3
            button3['command'] = lambda: download(id3, name3, singer3)
        if i == 3:
            id4 = str(music_ids[i])
            name4 = str(music_names[i])
            singer4 = str(music_singers[i][0])
            value4.set(name4 + '   ' + singer4)
            entry4['textvariable'] = value4
            button4['command'] = lambda: download(id4, name4, singer4)
        if i == 4:
            id5 = str(music_ids[i])
            name5 = str(music_names[i])
            singer5 = str(music_singers[i][0])
            value5.set(name5 + '   ' + singer5)
            entry5['textvariable'] = value5
            button5['command'] = lambda: download(id5, name5, singer5)


# 没有搜索之前点击下载按钮的提示
def tishi():
    messagebox.showinfo('提示', '请先进行搜索')


# 点击下载按钮执行(有点多余,可以去掉直接用get_downloadUrl)
def download(id, name, singer):
    sourse = v.get()
    get_downloadUrl(id, name, singer, sourse)


if __name__ == '__main__':
    # get_musicInfo('嘲笑声','tencent')
    # get_downloadUrl('0030tRLQ1e4mCn','嘲笑声','Big Daddy','tencent')

    root = Tk()
    win_width = root.winfo_screenwidth()
    win_height = root.winfo_screenheight()
    root.geometry('500x400+' + str(int(win_width / 2 - 250)) + '+' + str(int(win_height / 2 - 200)))
    root.minsize(500, 400)
    root.maxsize(500, 400)
    root.title('音乐下载器-敲出一片天')
    # get_downloadUrl('64561','单车(Live)','陈奕迅')

    query = StringVar()
    query.set('歌名+歌手更准确哦')

    # entry的参数:https://www.cnblogs.com/monsteryang/p/6575877.html

    entry = Entry(root, width=21, font=('隶书', 20), foreground='orange',
                  borderwidth=3, insertbackground='red', textvariable=query)
    entry.place(relx=0.05, rely=0.1)

    button = Button(root, width=8, text='搜索', font=('隶书', 18), bg='orange', fg='white', command=search_music)
    button.place(relx=0.7, rely=0.09)

    v = StringVar()
    v.set('netease')
    r1 = Radiobutton(text='网易', value='netease', font=('隶书', 18), fg='orange', variable=v)
    r2 = Radiobutton(text='qq', value='tencent', font=('隶书', 18), fg='orange', variable=v)
    r3 = Radiobutton(text='酷狗', value='kugou', font=('隶书', 18), fg='orange', variable=v)
    r4 = Radiobutton(text='百度', value='baidu', font=('隶书', 18), fg='orange', variable=v)
    r1.place(relx=0.08, rely=0.2)
    r2.place(relx=0.28, rely=0.2)
    r3.place(relx=0.48, rely=0.2)
    r4.place(relx=0.68, rely=0.2)

    frame = Frame(root, height=250, width=420, bd=1, relief="groove", bg='gray')
    frame.place(relx=0.06, rely=0.3)

    value1 = StringVar()
    entry1 = Entry(frame, width=21, font=('隶书', 15), bg='gray', relief="flat",
                   borderwidth=3, textvariable=query)
    entry1.place(relx=0.05, rely=0.04)
    button1 = Button(frame, width=8, text='下载', font=('隶书', 12), bg='gray', fg='black', command=tishi)
    button1.place(relx=0.7, rely=0.04)

    value2 = StringVar()
    entry2 = Entry(frame, width=21, font=('隶书', 15), relief="flat", bg='gray',
                   borderwidth=3, textvariable=query)
    entry2.place(relx=0.05, rely=0.24)
    button2 = Button(frame, width=8, text='下载', font=('隶书', 12), bg='gray', fg='black', command=tishi)
    button2.place(relx=0.7, rely=0.24)

    value3 = StringVar()
    entry3 = Entry(frame, width=21, font=('隶书', 15), bg='gray', relief="flat",
                   borderwidth=3, textvariable=query)
    entry3.place(relx=0.05, rely=0.44)
    button3 = Button(frame, width=8, text='下载', font=('隶书', 12), bg='gray', fg='black', command=tishi)
    button3.place(relx=0.7, rely=0.44)

    value4 = StringVar()
    entry4 = Entry(frame, width=21, font=('隶书', 15), bg='gray', relief="flat",
                   borderwidth=3, textvariable=query)
    entry4.place(relx=0.05, rely=0.64)
    button4 = Button(frame, width=8, text='下载', font=('隶书', 12), bg='gray', fg='black', command=tishi)
    button4.place(relx=0.7, rely=0.64)

    value5 = StringVar()
    entry5 = Entry(frame, width=21, font=('隶书', 15), bg='gray', relief="flat",
                   borderwidth=3, textvariable=query)
    entry5.place(relx=0.05, rely=0.84)
    button5 = Button(frame, width=8, text='下载', font=('隶书', 12), bg='gray', fg='black', command=tishi)
    button5.place(relx=0.7, rely=0.84)

    root.mainloop()

3. b站视频下载

import requests
import re
import json
from tkinter import *
from tkinter import messagebox


# 获得播放页面代码,获取我们需要的数据，转为json数据
def get_html_one(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'
    }
    response = requests.get(url, headers=headers)
    try:
        title = re.findall(r'<title data-vue-meta="true">(.+)_.+</title>', response.text)
        response = re.search(r'"data":.+,"session"', response.text)
        text = response.group()
        text = json.loads(text[7:-10])
        video_url = text['dash']['video'][0]['baseUrl']
        audio_url = text['dash']['audio'][0]['baseUrl']
        return video_url, audio_url, title[0]
    except:
        print('该视频不支持下载')
        info.set('该视频不支持下载')
        messagebox.showinfo('提示', '该视频不支持下载')
        return None


# 下载合集
def get_html_more(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'
    }
    response = requests.get(url, headers=headers)
    title = re.findall(r'<title data-vue-meta="true">(.+)_.+</title>', response.text)
    video_title.set(title[0])
    response = re.search(r'window.__INITIAL_STATE__=.+;(function', response.text)
    text = json.loads(response.group()[25:-10])
    cids = []
    names = []
    for info in text['videoData']['pages']:
        cids.append(str(info['cid']))
        names.append(info['part'])
    return cids, names


# 下载视频和音频到本地
def download_one(video_url, audio_url, title):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0',
        'Referer': 'https://www.bilibili.com/video/',
        'Origin': 'https://www.bilibili.com',
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate, sdch, br',
        'Accept-Language': 'zh-CN,zh;q=0.8'
    }
    print(title + ' 开始下载')

    try:
        video_response = requests.get(video_url, headers=headers)
        audio_response = requests.get(audio_url, headers=headers)
        with open(title + '.mp4', 'wb') as f:
            f.write(video_response.content)
        with open(title + '.mp3', 'wb') as f:
            f.write(audio_response.content)
    except:
        print(title + ' 下载失败')
        info.set(title + ' 下载失败')
        messagebox.showinfo('抱歉', title + ' 下载失败')
        return
    print(title + ' 下载完成')
    info.set(title + ' 下载完成')
    messagebox.showinfo('恭喜', title + ' 下载完成')


# 下载合集
def download_more(cids, names, url):
    number = len(cids)
    for i in range(number):
        url = url + '?p{}'.format(i + 1)
        video_url, audio_url, title = get_html_one(url)
        download_one(video_url, audio_url, names[i])
        print('=========================================')


# 点击搜索
def serach():
    button1.config(state="active")
    baseurl = 'https://www.bilibili.com/video/{}'
    video_id = entry.get()
    url = baseurl.format(video_id)
    flag = v.get()
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'
    }
    try:
        response = requests.get(url, headers=headers)
        title = re.findall(r'<title data-vue-meta="true">(.+)_.+</title>', response.text)
        if title[0] == '视频去哪了呢？':
            messagebox.showinfo('提示', '您输入的视频id不正确')
            return
        video_title.set(title[0])
        button1['command'] = lambda: download(url, flag)
    except:
        messagebox.showinfo('提示', '您输入的视频id不正确')
        return


# 点击下载
def download(url, flag):
    button1.config(state="disable")
    if flag == 0:
        video_url, audio_url, title = get_html_one(url)
        if video_url == None:
            return
        download_one(video_url, audio_url, title)
    else:
        cids, names = get_html_more(url)
        download_more(cids, names, url)

    print('下载完成,感谢您的使用')
    info.set('下载完成,感谢您的使用')


def tishi():
    messagebox.showinfo('提示', '请先进行搜索')


if __name__ == '__main__':
    root = Tk()
    win_width = root.winfo_screenwidth()
    win_height = root.winfo_screenheight()
    root.geometry('400x270+' + str(int(win_width / 2 - 200)) + '+' + str(int(win_height / 2 - 135)))
    root.minsize(400, 250)
    root.maxsize(400, 250)
    root.title('小破站下载器-敲出一片天')

    video_id = StringVar()
    video_id.set('请输入视频ID')

    entry = Entry(root, width=19, font=('隶书', 20), foreground='orange',
                  borderwidth=3, insertbackground='red', textvariable=video_id)
    entry.place(relx=0.02, rely=0.1)

    button = Button(root, width=7, text='搜索', font=('隶书', 18), bg='orange', fg='white', command=serach)
    button.place(relx=0.72, rely=0.09)

    v = IntVar()
    v.set(0)
    r1 = Radiobutton(text='单个视频', value=0, font=('隶书', 18), fg='orange', variable=v)
    r2 = Radiobutton(text='视频合集', value=1, font=('隶书', 18), fg='orange', variable=v)
    r1.place(relx=0.05, rely=0.25)
    r2.place(relx=0.45, rely=0.25)

    video_title = StringVar()
    video_title.set('视频标题')
    entry1 = Entry(root, width=30, font=('隶书', 15), fg='black', bg='#F0F0F0', relief='flat',
                   borderwidth=3, insertbackground='red', textvariable=video_title)
    entry1.place(relx=0.06, rely=0.4)

    button1 = Button(root, width=8, text='开始下载', font=('隶书', 12), bg='gray', fg='black', command=tishi)
    button1.place(relx=0.7, rely=0.4)

    info = StringVar()
    info.set('下载结果')
    entry_info = Entry(root, width=30, font=('隶书', 15), fg='red', bg='#F0F0F0', relief='flat',
                       borderwidth=3, textvariable=info)
    entry_info.place(relx=0.2, rely=0.6)

    label = Label(root, text='下载过程可能会出现无响应情况
下载完就好了', width=30, font=('隶书', 15), fg='black', bg='#F0F0F0',
                  relief='flat',
                  borderwidth=3)
    label.place(relx=0.06, rely=0.8)

    root.mainloop()

4.python爬虫框架scrapy爬取B站排行榜数据并保存到MongoDB数据库

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class BiliItem(scrapy.Item):
    # define the fields for your item here like:
    _id = scrapy.Field()
    title = scrapy.Field()
    play_num = scrapy.Field()
    up_name = scrapy.Field()
    score = scrapy.Field()

bili.py

# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
import scrapy
from bilibili.bili.bili.items import BiliItem


class BiliRankeSpider(scrapy.Spider):
    name = 'bili_ranke'
    allowed_domains = ['bilibili.com']
    start_urls = ['https://www.bilibili.com/ranking/all/0/0/3']

    def parse(self, response):
        titles = response.xpath('//div[@class="info"]//a[@class="title"]/text()').extract()
        play_nums = response.xpath('//div[@class="detail"]/span[@class="data-box"][1]/text()').extract()
        up_names = response.xpath('//div[@class="detail"]/a/span[@class="data-box"][1]/text()').extract()
        scores = response.xpath('//div[@class="pts"]/div/text()').extract()

        for title, play_num, up_name, score in zip(titles, play_nums, up_names, scores):
            item = BiliItem()
            item['title'] = title
            item['play_num'] = play_num
            item['up_name'] = up_name
            item['score'] = score
            yield item

pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo

class MoviesPipeline(object):

    def open_spider(self, spider):
        self.client = pymongo.MongoClient()

    def process_item(self, item, spider):
        self.client.bilibili.ranke.insert_one(item)
        return item

    def close_spider(self, spider):
        self.client.close()

附（MongoDB数据库python基本操作）

import pymongo

# 连接数据库
# 默认
client = pymongo.MongoClient()
# 自定义
# client = pymongo.MongoClient('ip',port)

# 选择实例（数据库）
person = client.person
# 选择集合（表）
student = person.student

#操作数据
# 查找所有信息
# result = student.find()
# for r in result:
#     print(r)

# print(result.next())

# 筛选
# result = student.find({"age":20})
# for r in result:
#     print(r)

# 排序
# result = student.find().sort("age",1)
# result = student.find().sort("age",pymongo.ASCENDING)
# for r in result:
#     print(r)

# 分页（偏移）
# result = student.find().limit(3)
# for r in result:
#     print(r)
#
#
# result = student.find().limit(3).skip(2)
# for r in result:
#     print(r)

# 统计
# result = student.find().count()
# print(result)

# 增加数据
# data = {"name":'曾强','age':22}
# student.insert(data)
# result = student.count()
# print(result)

# 删除数据
# data = {"name":'zq2','age':20}
# student.remove(data)

# 更新
data = {"name":"zq1"}
result = student.find_one(data)
print(result)
result["country"]="中国"
student.update(data,{'$set':result})

以上项目我都在bilibili上录有视频，看不明白可以去看一下视频，我的B站名：敲出一片天_bili