python之简单爬虫

#爬虫 ：模拟浏览器发送请求，下载网页代码，只提取有用的数据，存放于数据库或者文件中
#方案：python语法来实现，requests库发请求解析请求，excel表存储数据
#爬虫操作流程：1.模拟浏览器发送请求
             #2.下载网页代码
             #3.只提取有用的数据
             #4.存放于数据库或者文件中

#--------------------------------------------------------------------------------
#excel初始化：新建一个excel文件，创建个子表
#导入模块
import requests,re
import xlwt
workBook = xlwt.Workbook(encoding='uft-8')#此时还在缓存里
#创建子表
workSheet = workBook.add_sheet('51job')
#创建表头
colName = ['岗位名称','公司名称','地点','薪资','发布时间']
#把表头写进表格中
for one  in range(0,len(colName)):#0,1,2,3,4
    #写进单元格
    workSheet.write(0,one,colName[one]) #行号，列号，内容
#--------------------------------------------------------------------------------

#获取网站的页数,定义个函数
# <span class="td">共7页，到第</span>
def get_pagenum():
    web_url = 'https://search.51job.com/list/080200,000000,0000,00,9,07,%25E8%25BD%25AF%25E4%25BB%25B6%25E6%25B5%258B%25E8%25AF%2595,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
    resp = requests.get(web_url)  # 构建请求
    resp.encoding = 'gbk'  # 解决乱码
    pages = int(re.findall('<span class="td">共(.*?)页，到第</span>', resp.text, re.S)[0])
    return pages
#--------------------------------------------------------------------------------

#目标：获取符合要求的岗位
raw = 1 #内容的初始化行=1 ，因为有表头了
#1.构建请求
for one in range(1,get_pagenum()+1):
    web_url = f'https://search.51job.com/list/080200,000000,0000,00,9,07,%25E8%25BD%25AF%25E4%25BB%25B6%25E6%25B5%258B%25E8%25AF%2595,2,{one}.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare='
    resp = requests.get(web_url) # 构建请求
    resp.encoding = 'gbk' #解决乱码
    # print(resp.request.headers) #请求头
    # print(resp.request.body) #请求体
    #2.解析响应数据
    print(resp.text)
    #3.提取有效数据
    #从响应数据中提取有效数据。re.S #使.匹配包括换行符在内的所有字符
    info = re.findall('<div class="el">(.*?)</div>',resp.text,re.S) #数据中有很多换行符，需要加re.S
    for line in info:
        #1.获取岗位名称
        temp = re.findall('<a target="_blank" title="(.*?)" href',line,re.S)
        # 因为re.findall返回值是列表，所以要取值就必须写下标
        jobName = temp[0].strip()
        workSheet.write(raw, 0, jobName)
        # 2.获取公司名称，#因为re.findall返回值是列表，所以要取值就必须写下标
        company = temp[1].strip()
        workSheet.write(raw, 1, company)
        # 3.获取地点，#因为re.findall返回值是列表，所以要取值就必须写下标
        adress = re.findall('<span class="t3">(.*?)</span>',line,re.S)[0]
        workSheet.write(raw, 2, adress)
        # 4. 获取薪资，#因为re.findall返回值是列表，所以要取值就必须写下标
        salary = re.findall('<span class="t4">(.*?)</span>', line, re.S)[0]
        workSheet.write(raw, 3, salary)
        #5.获取发布时间
        jobTime = re.findall('<span class="t5">(.*?)</span>', line, re.S)[0]
        workSheet.write(raw, 4, jobTime)
        raw +=1
        print(jobName,company,adress,salary,jobTime)

#--------------------------------------------------------------------------------

#4.存储数据
#保存
workBook.save('D:\51.job.xls')

#--------------------------------------------------------------------------------








'''
<div class="el">
        <p class="t1 ">
            <em class="check" name="delivery_em" onclick="checkboxClick(this)"></em>
            <input class="checkbox" type="checkbox" name="delivery_jobid" value="85295957" jt="0" style="display:none">
            <span>
                <a target="_blank" title="软件测试工程师" href="https://jobs.51job.com/hangzhou-yhq/85295957.html?s=01&amp;t=0" onmousedown="">
                    软件测试工程师                </a>
            </span>
                                                                    </p>
        <span class="t2"><a target="_blank" title="杭州老板电器股份有限公司" href="https://jobs.51job.com/all/co2322348.html">杭州老板电器股份有限公司</a></span>
        <span class="t3">杭州-余杭区</span>
        <span class="t4">10-15万/年</span>
        <span class="t5">03-21</span>
    </div>

'''