2019.10.24

# -*- coding: utf-8 -*-
import scrapy
from ..items import JobscrawlerQianchengwuyouItem
import datetime

class QianchengSpiderSpider(scrapy.Spider):
    name = 'qiancheng_spider'
    # allowed_domains = ['qq.com']
    start_urls = [
        # 数据分析师
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
        # 人工智能
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E4%25BA%25BA%25E5%25B7%25A5%25E6%2599%25BA%25E8%2583%25BD,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
        # 算法工程师
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E7%25AE%2597%25E6%25B3%2595%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
        # 深度学习
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%25B7%25B1%25E5%25BA%25A6%25E5%25AD%25A6%25E4%25B9%25A0,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
        # 数据挖掘
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E6%258C%2596%25E6%258E%2598,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=',
        # 机器学习
        'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%259C%25BA%25E5%2599%25A8%25E5%25AD%25A6%25E4%25B9%25A0,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='
    ]
    start_url_tags=[
        "数据分析师",
        "人工智能",
        "算法工程师",
        "深度学习",
        "数据挖掘",
        "机器学习",
    ]

    def __init__(self):
        self.record_date = datetime.datetime.now().strftime('%Y-%m-%d')

    def start_requests(self):
        for index in range(len(self.start_urls)):
            url = self.start_urls[index]
            tag = self.start_url_tags[index]
            yield scrapy.Request(url,callback=self.parse,meta={'tag':tag},dont_filter=True)

    def parse(self, response):
        tag = response.meta['tag']
        xpath = '//div[@class="el"]'
        items = response.xpath(xpath)
        for item in items:
            if not len(item.xpath('./p[@class="t1 "]')):
                continue
            url = item.xpath('./p[@class="t1 "]//a/@href').extract_first()
            title = item.xpath('./p[@class="t1 "]//a/text()').extract_first()
            if tag == '算法' and not ('算法' in title):
                continue
            yield scrapy.Request(url,callback=self.detail_parse,meta={'tag':tag},dont_filter=True)
        next_page_url = response.xpath('//a[@id="rtNext"]/@href').extract_first()
        if next_page_url is None:
            yield scrapy.Request(next_page_url, callback=self.parse,meta={'tag':tag},dont_filter=True)

    def detail_parse(self,response):
        item = JobscrawlerQianchengwuyouItem()
        item['job_tag'] = response.meta['tag']
        item['job_url'] = response.url
        item['record_date'] = self.record_date
        # 招聘名称、职位信息、薪资、职位福利、经验要求、学历要求
        item['job_name'] = response.xpath('//div[@class = "cn"]/h1/text()').extract_first().strip()
        item['job_info'] = "".join(response.xpath('//div[@class = "bmsg job_msg inbox"]//text()').extract()).strip()
        item['job_salary'] = "".join(response.xpath('//div[@class = "cn"]/strong/text()').extract()).strip()
        item['job_welfare'] = ",".join(response.xpath('//span[@class="sp4"]/text()').extract()).strip()
        item['job_exp_require'] = response.xpath('//p[@class="msg ltype"]/text()').extract()[1].strip()
        item['job_edu_require'] = response.xpath('//p[@class="msg ltype"]/text()').extract()[2].strip()
        # 公司名称、公司行业、公司性质、公司人数、公司地址、公司概况、公司融资阶段
        item['company_name'] = response.xpath('//div[@class = "com_msg"]//p/text()').extract_first().strip()
        item['company_industry'] = "".join(response.xpath('//span[@class = "i_trade"]/..//text()').extract()).strip()
        item['company_nature'] = "".join(response.xpath('//span[@class = "i_flag"]/../text()').extract()).strip()
        item['company_people'] = "".join(response.xpath('//span[@class = "i_people"]/../text()').extract()).strip()
        item['company_location'] = ""
        item['company_overview'] = "".join(response.xpath('//div[@class = "tmsg inbox"]//text()').extract()).strip()
        item['company_financing_stage'] = ""
        yield item