requests的基本使用

1. 简单获取一个页面信息

import requests

# requests可以通过requests的get、post、put等方法进行请求调用
r = requests.get('https://www.baidu.com/')
print(r)
print(type(r))
print(r.status_code)
print(type(r.text))
print(r.text)
print(r.cookies)

2. 通过params参数，get请求进行url字符串拼接

import requests

data = {
    'name':'dmr',
    'age':25,
}
# 将data传给参数params，会把url拼接成?key1=value1&key2=value2的形式
r = requests.get(url='http://httpbin.org/get', params=data)
print(r.text, r.url)
# r.text返回的是一个字典类型的json格式，通过json方法解析可以获取到一个字典
print(type(r.text))
print(r.json())
print(type(r.json()))

3. 抓取二进制数据（如图片等）

import requests

r = requests.get('https://dss0.bdstatic.com/5aV1bjqh_Q23odCf/static/superman/img/logo_top-e3b63a0b1b.png')
# r.text字符串格式；r.content字节格式
print(r.text, r.content)
print(type(r.text), type(r.content))
# 保存为文件格式
with open('baidu.png','wb') as f:
    f.write(r.content)

4. 添加headers请求头的User-agent进行页面访问

import requests

r = requests.get('https://www.zhihu.com/')
print(r.text)
headers = {
    'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
r2 = requests.get('https://www.zhihu.com/', headers=headers)
print(r2.text)

5. post请求,data字典内容，提交数据到表单

import requests

data = {
    'name':'dmr',
    'age':25,
}
r = requests.post('http://httpbin.org/post', data=data)
print(r.text)


print(dir(r))
print(dir(requests.codes))
print(requests.codes.ok)

6. 扩展用法

# 文件上传
import requests

files = {'file':open('baidu.png', 'rb')}
r = requests.post('http://httpbin.org/post', files=files)
print(r.text)

# Cookies
import requests

r = requests.get('https://www.baidu.com/')
print(r.cookies, r.cookies.items())
for key, value in r.cookies.items():
    print(key + '=' + value)


# 设置headers的Cookie值来保持登录状态
import requests

headers = {
    'Cookie':'JSNID=2D7B55dfsdfsdfECE6E47C6AD; signature=MjEyMzJmMjk3YdsfsdfNzQzODk0YTBlNGE4MDFmYzMxNTgzOTE4MTk4NjMx3cf711642ab38b713e3e9a54f19fad0e; timeStamp=da23b847-43c0-46ab-ba78-9e8ca41a11ea; redisKey=5f948ef7-ae9a-4d67-a7f8-4211baafe92f',
    'User-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
r = requests.get('http://ceshi/public/html/index.html', headers=headers)
print(r.text)


# 通过cookies参数key和value来保持登录状态
import requests

cookies = 'JSEID=2D7B5505568DCB699685BECE6E47C6AD; signare=MjEyMzJmMjk3YTDk0YTBlNGE4MDFmYzMxNTgzOTE4MTk4NjMx3cf711642ab38b713e3e9a54f19fad0e; timeStamp=da23b847-43c0-46ab-ba78-9e8ca41a11ea; redisKey=5f948ef7-ae9a-4d67-a7f8-4211baafe92f'
headers = {
    'User-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
jar = requests.cookies.RequestsCookieJar()
for cookie in cookies.split(';'):
    key, value = cookie.strip().split('=')
    print(key, value)
    jar.set(key, value)
r = requests.get('http://ceshi/public/html/index.html',headers=headers,cookies=jar)
print(r.text)


## 保持会话，Session
import requests

# 给测试页面设置值
r = requests.get('http://httpbin.org/cookies/set/dmr/25')
# 独立进行第二次访问页面时，返回的页面内容为空
r2 = requests.get('http://httpbin.org/cookies')
print(r.text, r2.text)

# 通过session保持会话进行二次访问
s = requests.Session()
# 给测试页面设置值
r = s.get('http://httpbin.org/cookies/set/dmr/25')
# 通过session保持会话进行二次访问时，设置的值还存在
r2 = s.get('http://httpbin.org/cookies')
print(r.text, r2.text)


## SSL证书验证，verify是否检查证书验证，默认为true
import requests

r = requests.get('https://www.12306.cn')
r2 = requests.get('https://www.12306.cn', verify=False)
r3 = requests.get('https://www.12306.cn', cert=('path/cert.crt', '/path/key'))

print(r.text)


## proxy，代理设置
import requests

# 无效的代理
proxies = {
    'http':'http://10.0.0.100:3128',
    'https':'http://10.0.0.100:3168',
}
requests.get('https://www.taobao.com', proxies=proxies)


# 超时设置
 import requests

 r = requests.get('http://www.python.org', timeout=0.5)
 print(r.status_code)


 ## 弹窗身份认证
 import requests
 from requests.auth import HTTPBasicAuth

 auth = HTTPBasicAuth('username', 'password')
 r = requests.get('http://localhost:5000', auth=auth)
 # 可简写，requests默认使用HTTPBasicAuth来进行认证
 r2 = requests.get('http://localhost:5000', auth=('username', 'password'))
 print(r.status_code, r2.status_code)

7. requests将参数内容封装成数据对象来请求访问页面

from requests import Request, Session

url = 'https://www.baidu.com'
data = {
    'name': 'dmr',
    'age': 25
}
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
}
# 构建Request请求对象
req = Request(method='POST', url=url, data=data, headers=headers)
# 生成会话
s = Session()
# 构建prepared数据结构请求对象
prepared = s.prepare_request(req)
# 发送请求
r = s.send(prepared)
print(r.text, r.status_code)