Scrapy实现腾讯招聘网信息爬取【Python】

一.腾讯招聘网

二.代码实现

　　1.spider爬虫

# -*- coding: utf-8 -*-
import scrapy
from Tencent.items import TencentItem

class TencentSpider(scrapy.Spider):
    name = 'tencent'
    allowed_domains = ['tencent.com']
    base_url = 'https://hr.tencent.com/position.php?&start='
    offset = 0
    start_urls = [base_url + str(offset)]
    def parse(self, response):
        node_list = response.xpath("//tr[@class='even'] | //tr[@class='odd']")
        for node in node_list:
            # 创建Item对象来保存信息
            item = TencentItem()
            positionName = node.xpath("./td[1]/a/text()").extract()[0]
            if len(node.xpath("./td[2]/text()")):
                positionType = node.xpath("./td[2]/text()").extract()[0]
            else:
                positionType = ""
            positionNumber = node.xpath("./td[3]/text()").extract()[0]
            location = node.xpath("./td[4]/text()").extract()[0]
            publishTime = node.xpath("./td[5]/text()").extract()[0]

            # 保存到item中
            item['positionName'] = positionName
            item['positionType'] = positionType
            item['positionNumber'] = positionNumber
            item['location'] = location
            item['publishTime'] = publishTime

            yield item

        # 判断是否需要拼接下一页路径
        if self.offset < 2770:
            self.offset += 10
            # 拼接下一页路径
            url = self.base_url + str(self.offset)
            # dont_filter=True 禁止因域名不同而过滤
            yield scrapy.Request(url, callback=self.parse)

　　2.管道

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json

class TencentPipeline(object):
    def __init__(self):
        self.f = open("Tencent.json", "w")

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii=False) + ",
"
        self.f.write(text)
        return item

    def close_spider(self, spider):
        self.f.close()

　　3.实体

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TencentItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 职位名称
    positionName = scrapy.Field()
    # 职位类别
    positionType = scrapy.Field()
    # 人数
    positionNumber = scrapy.Field()
    # 地点
    location = scrapy.Field()
    # 发布时间
    publishTime = scrapy.Field()

三.结果【部分展示】

{"positionName": "15605-动作RPG手游游戏活动策划", "positionType": "产品/项目类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "21645-高级法律顾问", "positionType": "职能类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "HGJ-senior legal counsel(MA)", "positionType": "职能类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "19837-电影新媒体媒介经理", "positionType": "市场类", "positionNumber": "1", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "HGJ-Legal Counsel (Technology Transactions)", "positionType": "职能类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22989-云服务平台部--服务运营中心总监", "positionType": "技术类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22989-经营平台产品中心web前端开发", "positionType": "技术类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22989-视频云高级Web前端开发", "positionType": "技术类", "positionNumber": "2", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "27554-腾讯音乐商业制片人（上海）", "positionType": "市场类", "positionNumber": "1", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "30361-天天P图图像处理后台开发（上海）", "positionType": "技术类", "positionNumber": "1", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游资深美术3D设计（上海）", "positionType": "设计类", "positionNumber": "1", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游3D动画设计师（上海）", "positionType": "设计类", "positionNumber": "1", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游3D特效美术师（上海）", "positionType": "设计类", "positionNumber": "1", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游交互设计师(上海)", "positionType": "设计类", "positionNumber": "1", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "AQ-产品安全经理（深圳）", "positionType": "职能类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "AQ-产品安全经理（广州）", "positionType": "职能类", "positionNumber": "1", "location": "广州", "publishTime": "2019-02-14"},
{"positionName": "29050-数据安全经理/专家（深圳）", "positionType": "职能类", "positionNumber": "2", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "29050-数据安全经理/专家（北京）", "positionType": "职能类", "positionNumber": "2", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "AQ-行业合作经理（北京）", "positionType": "职能类", "positionNumber": "1", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "AQ-行业合作经理（深圳）", "positionType": "职能类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "27553-腾讯音乐人曲库运营", "positionType": "内容编辑类", "positionNumber": "2", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22086-体育号创作平台产品经理", "positionType": "产品/项目类", "positionNumber": "1", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "22086-体育号CP管理产品经理 ", "positionType": "产品/项目类", "positionNumber": "1", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "22086-体育号内容质量产品经理", "positionType": "产品/项目类", "positionNumber": "1", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "28297-二次元手游本地化策划（深圳）", "positionType": "产品/项目类", "positionNumber": "2", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "GY0-腾讯云海外商务拓展", "positionType": "市场类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "PCG10-高级产品经理（工具产品方向）", "positionType": "产品/项目类", "positionNumber": "1", "location": "成都", "publishTime": "2019-02-14"},
{"positionName": "18432-策略分析师", "positionType": "产品/项目类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "18432-基金高级分析师", "positionType": "产品/项目类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "21309-在线教育运营专家/增长黑客", "positionType": "产品/项目类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "28481-高级医疗商务拓展经理（北京）", "positionType": "市场类", "positionNumber": "1", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "21882-高级医学编辑（深圳/北京）", "positionType": "内容编辑类", "positionNumber": "2", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "18402-MMO手游-平台渠道运营", "positionType": "产品/项目类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "28170-腾讯游戏直播业务管理经理（深圳）", "positionType": "产品/项目类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "28170-腾讯游戏直播内容品牌经理（深圳）", "positionType": "市场类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "PCG10-浏览器阅读中心后台开发工程师（深圳）", "positionType": "技术类", "positionNumber": "1", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "PCG10-浏览器阅读中心前端开发工程师（深圳）", "positionType": "技术类", "positionNumber": "2", "location": "深圳", "publishTime": "2019-02-14"},

 四.详情