python16_day36【爬虫1】

一、requests

　　1. GET请求

# 1、无参数实例
 
import requests
 
ret = requests.get('https://github.com/timeline.json')
 
print(ret.url)
print(ret.text)
 
 

# 2、有参数实例
 
import requests
 
payload = {'key1': 'value1', 'key2': 'value2'}
ret = requests.get("http://httpbin.org/get", params=payload)
 
print(ret.url)
print(ret.text)

　　2.POST请求

# 1、基本POST实例
 
import requests
 
payload = {'key1': 'value1', 'key2': 'value2'}
ret = requests.post("http://httpbin.org/post", data=payload)
 
print(ret.text)
 
 
# 2、发送请求头和数据实例
 
import requests
import json
 
url = 'https://api.github.com/some/endpoint'
payload = {'some': 'data'}
headers = {'content-type': 'application/json'}
 
ret = requests.post(url, data=json.dumps(payload), headers=headers)
 
print(ret.text)
print(ret.cookies)

　　3.其它请求　　

requests.get(url, params=None, **kwargs)
requests.post(url, data=None, json=None, **kwargs)
requests.put(url, data=None, **kwargs)
requests.head(url, **kwargs)
requests.delete(url, **kwargs)
requests.patch(url, data=None, **kwargs)
requests.options(url, **kwargs)
 
# 以上方法均是在此方法的基础上构建
requests.request(method, url, **kwargs)

　　4.汽车之家新闻

import requests
from bs4 import BeautifulSoup

# response = requests.get("http://www.autohome.com.cn/news/")
# # response.text 是str
# # response.content 是bytes二进制
#
# response.encoding = 'gbk'   # 网站使用了gbk
# root = BeautifulSoup(response.text, 'html.parser')   # 将返回结果拿到用bs解析
# outer_div_obj = root.find(name='div', id='auto-channel-lazyload-article')  # 打到div id='xx'
# li_obj_list = outer_div_obj.find_all(name='li')     # 拿到里面所有的LI
#
# for li_obj in li_obj_list:
#     if not li_obj.find('h3'):
#         continue
#     title_obj = li_obj.find('h3')       # 拿到对象 H3标签
#     summary_obj = li_obj.find('p')      # 拿到对象 P标签
#     img_obj = li_obj.find('img')        # 拿到对象 IMG标签
#     src = img_obj.attrs.get('src')      # 从IMG标签对象中拿到src属性
#
#     print(src, title_obj.text, summary_obj.text)

response = requests.get("http://www.autohome.com.cn/news/")
response.encoding = 'gbk'

soup = BeautifulSoup(response.text, 'html.parser')
tag = soup.find(name='div', attrs={'id': 'auto-channel-lazyload-article'})
li_list = tag.find_all('li')

for li in li_list:
    h3 = li.find('h3')

    if not h3:
        continue
    print("33[33;1m标题: {0}33[0m".format(h3.text))
    print("33[34;1m路径: http://{0}33[0m".format(li.find('img').attrs['src']))
    print("33[34;1m内容: {0}33[0m".format(li.find('p').text))

　　5.github登录

#!/usr/bin/env python
# -*-coding:utf8-*-
# __author__ = "willian"

import requests
from bs4 import BeautifulSoup
# 第一次请求: 获取 token and cookie
r1 = requests.get('https://github.com/login')
b1 = BeautifulSoup(r1.text, 'html.parser')
# get token
auth_token = b1.find(name='input', attrs={'name': 'authenticity_token'}).get('value')
# get cookies
r1_cookie_dict = r1.cookies.get_dict()


# 第二次请求: 发送用户认证
r2 = requests.post("https://github.com/session",
                   data={
                       'commit': "Sign in",
                       'utf8': '✓',
                       'authenticity_token': auth_token,
                       'login': '',
                       'password': ""
                   }, cookies=r1_cookie_dict)
# get cookies
r2_cookie_dict = r2.cookies.get_dict()

# 将两次的cookies合并
all_cookie_dict = {}
all_cookie_dict.update(r1_cookie_dict)
all_cookie_dict.update(r2_cookie_dict)


# 第三次请求:只有登录成功之后获取个人页面
r3 = requests.get('https://github.com/settings/emails', cookies=all_cookie_dict)
print(r3.text)

 　　6.抽屉点赞

#!/usr/bin/env python
# -*-coding:utf8-*-
# __author__ = "willian"

import requests
from bs4 import BeautifulSoup

# 1. 请求获取cookies
r0 = requests.get("http://dig.chouti.com")
r0_cookie_dict = r0.cookies.get_dict()


# 2. 授权
r1 = requests.post(
    url="http://dig.chouti.com/login",
    data={
        'phone': 'xx',
        'password': 'xx',
        'oneMonth': 1
    },
    cookies=r0_cookie_dict
)
r1_cookie_dict = r1.cookies.get_dict()

all_cookies = {}
all_cookies.update(r0_cookie_dict)
all_cookies.update(r1_cookie_dict)

# 3.点赞
r2 = requests.post(url='http://dig.chouti.com/link/vote?linksId=14808951', cookies=all_cookies)
print(r2.text)

二、Beautfulsoup4

三、wechat