Python爬虫（八）

源码：

import requests
import re
from my_mysql import MysqlConnect
import time,random


# 获取招聘详情链接
def get_urls(page, headers):
    url = 'https://hr.tencent.com/position.php?lid=&tid=&keywords=python&start=page'.format(page)
    response = requests.get(url, headers=headers)
    pat = r'href="(position_detail.*?)">'
    url_list_bytes = re.findall(pat.encode('utf-8'), response.content)
    return url_list_bytes

# 获取招聘详情
def get_info(url, headers):
    response = requests.get(url, headers=headers)
    html_bytes = response.content
    # print(html_bytes)

    # title 标题
    pat = r'id="sharetitle">(.*?)</td>'
    res = re.search(pat.encode('utf-8'), html_bytes)
    title = res.group(1).decode('utf-8')
    # address 地点
    pat = r'工作地点：</span>(.*?)</td>'
    res = re.search(pat.encode('utf-8'), html_bytes)
    address = res.group(1).decode('utf-8')
    # types 类别
    pat = r'职位类别：</span>(.*?)</td>'
    res = re.search(pat.encode('utf-8'), html_bytes)
    types = res.group(1).decode('utf-8')
    # counts 人数
    pat = r'招聘人数：</span>(.*?)</td>'
    res = re.search(pat.encode('utf-8'), html_bytes)
    counts = res.group(1).decode('utf-8')
    # duty 职责
    pat = r'工作职责.*?<ul class="squareli">(.*?)</ul>'
    res = re.search(pat.encode('utf-8'), html_bytes)
    duty_str = res.group(1).decode('utf-8')
    pat = r'<li>(.*?)</li>'
    duty = re.findall(pat,duty_str)
    duty = ('
').join(duty)
    # requires 要求
    pat = r'工作要求.*?<ul class="squareli">(.*?)</ul>'
    res = re.search(pat.encode('utf-8'), html_bytes)
    requires_str = res.group(1).decode('utf-8')
    pat = r'<li>(.*?)</li>'
    requires = re.findall(pat, requires_str)
    requires = ('
').join(requires)
    return title,address,types,counts,duty,requires


if __name__ == '__main__':
    mc = MysqlConnect('127.0.0.1','root','123456','homework')
    sql = "insert into tencentzp(title,address,types,counts,duty,requires) values(%s,%s,%s,%s,%s,%s)"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    for page in range(0,200,10):
        url_list_bytes = get_urls(page,headers)
        # print(url_list_bytes)
        for url in url_list_bytes:
            # print(url.decode('utf-8'))
            url = 'https://hr.tencent.com/' + url.decode('utf-8')
            info = get_info(url,headers)
            print(info)
            mc.exec_data(sql,info)
            time.sleep(random.random()*5)