第四节 request模拟浏览器

import urllib.request
import urllib.parse


#拉钩网，反爬虫策略很好，请求头需要添加Referer（引用; 引用页; 推荐人; 参照页），请求对象需要data=，method='POST'
# url = 'https://baike.baidu.com/item/%E8%99%9A%E6%8B%9F%E7%8E%AF%E5%A2%83/6529153?fr=aladdin'
url = 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
    'Referer':'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
}
data = {
    'first':"true",
    'pn':1,
    'kd':'python'
}
#调用urllib.request中的Request类，创建一个req对象
req = urllib.request.Request(url,headers=headers,data=urllib.parse.urlencode(data).encode('utf-8'),method='POST')
#将创建的req对象传给urllib.request.urlopen
resp = urllib.request.urlopen(req)
print(resp.read().decode('utf-8'))