爬虫知识随记

备忘

import urllib.parse
import urllib.request
#  将数据使用urlencode编码处理后，再使用encoding设置为utf-8编码
data = bytes(urllib.parse.urlencode({"word":"hello"}), encoding="utf-8")
response = urllib.request.urlopen("http://httpbin.org/post",data=data)    # 打开指定需要爬取的网页
html = response.read()    # 读取网页
print(html)    # 打印读取内容

import requests
data = {"word":"hello"}    # 表单参数
response = requests.post("http://httpbin.org/post",data=data)
print("状态码：",response.status_code)
# print("请求url：",response.url)
# print("头部信息：",response.headers)
# print("cookies：",response.cookies)
# print("文本形式的网页源码：",response.text)
print("字节流形式的网页源码：",response.content)

import requests
url = "https://book.douban.com//tag/营销"    # 网络请求地址
headers = {"User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit"
                        "/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"}
response = requests.get(url, headers=headers)    # 发送网络请求
print(response.content.decode("utf-8"))    # 以字节流形式打印网页源码

import requests
# 导入requests.exceptions中的三种异常类
from requests.exceptions import ReadTimeout,HTTPError,RequestException
# 循环发送请求50次
for a in range(0,50):
    try:    # 捕获异常
        url = "https://www.baidu.com/"    # 网页请求地址
        response = requests.get(url, timeout=0.06)    # 设置超时为0.06s
        print(response.status_code)    # 打印状态码
    except ReadTimeout:
        print("timeout")
    except HTTPError:
        print("httperror")
    except RequestException:
        print("reqeerror")

import requests
# 设置代理IP
proxy = {"http://":"60.188.90.33:3000",
         "https://":"183.128.240.228:6666"}
response = requests.get("https://www.baidu.com/", proxies = proxy)
print(response.content.decode("utf-8"))

import requests
from bs4 import BeautifulSoup
response = requests.get("http://news.baidu.com/")    # 发送网络请求
bs = BeautifulSoup(response.text,"lxml")    # 通过lxml解析器来解析文本形式的网页源码
print(bs.find("title").text)    # 找到title并以文本形式打印

# 任务：将爬取的内容写到本地
import requests
url = "https://www.bilibili.com/"
def use_requests(url):
    response = requests.get(url)
    # print(response.text)
    file_path = "E:/PyCharmProjects/SpyderStudy/哔哩哔哩首页.html"
    with open(file_path,"w",encoding="utf-8") as f:
        f.write(response.text)
if __name__ == "__main__":
    use_requests(url)


file_path = "E:/PyCharmProjects/SpyderStudy/哔哩哔哩_selenium_chrome.html"
with open(file_path,"w",encoding="utf-8") as f:
    f.write(data)

driver.close()

import urllib.request
import requests
from bs4 import BeautifulSoup
headers = {"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 "
                         "(KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"}
key = "小说"
key_ASCII = urllib.request.quote(key)
url = "https://book.douban.com/tag/" + str(key_ASCII) + "?start=0&type=T"
response = requests.get(url, headers=headers)
bs = BeautifulSoup(response.text, "lxml")
a = bs.select("#subject_list > ul > li:nth-child(1) > div.info > h2 > a")
a_1 = a[0]
print(a_1.get_text().replace("
", "").replace(" ", ""))

import urllib.request
import requests

headers = {"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 "
                             "(KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36"}

key = "华语"
key_ASCII = urllib.request.quote(key)
url = "https://movie.douban.com/j/search_subjects?type=movie&tag=" + str(
    key_ASCII) + "&sort=recommend&page_limit=20&page_start=0"
response = requests.get(url, headers=headers)
print(response)