爬虫相关基础知识梳理

1.PyQuery

from pyquery import PyQuery as pq

# url初始化
# html = ''
# doc = pq(html)
url = 'https://www.baidu.com'
doc = pq(url=url)
print(doc('head'))

# 文件初始化
doc = pq(filename='xxx.html')
print(doc('li'))

#　基本css选择器
doc('#id .class a') # id 下　class 的　li 标签
doc('#id .class.a') # 并列表示同级

# 查找元素
item = doc('.class')
lis = item.find('list')  #　子元素
lis = item.children() #　直接子元素　不常用
lis = item.children('.class')

lis = item.parent()   #父元素
lis = item.parents()  #祖先节点
lis = item.parents('li')

item.siblings()  #兄弟节点
item.siblings('')  #兄弟节点


# 遍历
lst = doc('li').items() #生成器
for ls in lst:
    pass

#　获取属性
lis.attr('href')
lis.attr.href

#　获取文本
lis.text()

#　获取html
lis.html()

#　dom操作
lis.remove_class('.class')
lis.add_class('.class')

lis.attr('name','link')  #　加一个ｎａｍｅ='link'　的属性
lis.css('font-size','14px') # 加一个css属性

lis.find('p').remove()  # 删除p标签

# 伪类选择器
doc('li:first-child')  #　第一个元素
doc('li:last-child')  #　最后一个元素
doc('li:child(2)')  #　第二个元素
doc('li:gt(2)')  #　第二个元素以上
doc('li:nth-child(2n)')  #　偶数元素
doc('li:contains(second)') #　包含second　文本的

2.requests

import requests
url = 'https://www.baidu.com'
resp = requests.get(url)
print(resp.cookies)
print(resp.text)

# get

data = {
    '':'',
    '':''
}
resp = requests.get(url,params=data)
# 解析json
print(resp.json()) #print(json.loads(resp.text)

# 获取二进制数据
print(resp.content)
with open('','wb') as f:
    f.write(resp.content)

# 添加headers
headers = {'User-Agent':''}
resp = resp.get(url,headers=headers)



# post
data = {}
resp = requests.post(url,data=data)
resp = requests.post(url,data=data,headers=headers)


# >>>高级操作
#   1.文件上传
files = {'file':open('','rb')}
resp = requests.post(url,files=files)
#   2.获取cookie4
for key,value in resp.cookies.items():
    print(key+'='+value)
#   3.会话维持
import requests
# requests.get('https://httpbin.org/cookies/set/number/12346789')
# resp = requests.get('https://httpbin.org/cookies')
s = requests.Session()
s.get('https://httpbin.org/cookies/set/number/12346789')
resp = s.get('https://httpbin.org/cookies')

#   4.证书验证
import requests
resp = requests.get('https://www.12306.cn',verify=False)
resp = requests.get('https://www.12306.cn',cert=('/path/server,crt','/path/key'))

#   5.代理设置
import requests
proxies = {
    'http':'http://127.0.0.1:9473',
    'https':'https://127.0.0.1:9473',
    'http':'http://uesr:password@127.0.0.1:9473' #带有用户名密码的代理
}
resp = requests.get(url,proxies=proxies)


#   6.认证设置
import requests
from requests.auth import HTTPBasicAuth
resp = requests.get(url,auth=HTTPBasicAuth('user','123'))


import requests
resp = requests.get(url,auth=('',''))


#   7.异常处理
from requests.exceptions import ReadTimeout,ConnectionError,RequestException
try:
    pass
except ReadTimeout:
    pass
except ConnectionError:
    pass
except RequestException:
    pass

4.selenium

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait as wdw

url = 'https://www.baidu.com'
browser = webdriver.Chrome()
try:
    browser.get(url)
    input = browser.find_element_by_id('kw')
    input.send_keys('Python')
    input.send_keys(Keys.ENTER)
   # input.clear()
    wait = wdw(browser,10)
    wait.until(EC.presence_of_element_located((By.ID,'content_left')))
    print(browser.current_url)
    print(browser.get_cookies())
    print(browser.page_source)
finally:
    browser.close()


# 声明浏览器对象
browser = webdriver.Chrome()
browser = webdriver.Firefox()
browser = webdriver.Edge()
browser = webdriver.PhantomJS()
browser = webdriver.Safari()

# 查找元素
browser.find_element_by_id('q')
browser.find_element_by_css_selector('#q')
browser.find_element_by_xpath('//*[@id="q"]')
browser.find_element('By.ID','q')

# 多个元素
browser.find_elements(By.CSS_SELECTOR,'.class li')
browser.find_elements_by_css_selector('.class li')

# 元素交互操作
button = browser.find_element_by_class_name('')
button.click()

#　交互动作
from selenium.webdriver import ActionChains
browser = webdriver.Chrome()
url = ''
browser.get(url)
browser.switch_to('')
source = browser.find_element_by_css_selector('#')
target = browser.find_element_by_css_selector('#')
actions = ActionChains(browser)
actions.drag_and_drop(source,target)
actions.perform()

# 执行javaScript
browser.execute_script('alert()')

# 获取元素信息
logo = browser.find_element_by_css_selector('#')
logo.get_attribute('class')

# 获取文本值
logo.text()

#　获取id　位置　　标签名　大小　

logo.location
logo.id
logo.tag_name
logo.size

# Frame
from selenium.common.exceptions import NoSuchElementException

browser.switch_to.frame('')
browser.switch_to.parent_frame('')

# 等待
# 隐式等待
browser.implicitly_wait(10) # 超出１０秒 异常

# 显示等待　常用　
wait = wdw(browser,)
wait.until(EC.presence_of_element_located((By.ID,'q')))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,'')))


# 前进　后退
browser.back()
browser.forward()

# cookies
browser.get_cookie()
browser.add_cookie()
browser.delete_all_cookies()

# 选项卡管理
browser.execute_script('windows.open()')  # 打开
browser.switch_to_window(browser.window_handles[1]) #切换

# 异常处理
from selenium.common.exceptions import TimeoutException,NoSuchElementException
try:
    pass
except TimeoutException:
    pass
except NoSuchElementException:
    pass
finally:
    browser.close()

4.re

import re

#  match--- 从起始位置开始匹配
content = 'Hello 123 4567 World_This is a Ragex Demo'
result = re.match('^Hellosdddsd{4}sw{10}.*$',content)
print(result)
print(result.group())
print(result.span())


#复用
pattern = re.compile('^Hello.*Demo$',re.S)
result = re.match(pattern,content)

#  泛匹配
result = re.match('^Hello.*Demo$',content)
# (d+)  .不能匹配换行符
# 贪婪匹配 .*   非贪婪匹配 .?


# search--- 返回第一个成功匹配的内容
# findall---  以列表形式返回能全部匹配的目标
# sub ---       替换
# compile ---   将正则字符串编译成正则表达式

# 尽量使用泛匹配,使用()得到匹配目标,尽量使用非贪婪模式,有换行符用re.S,能用search就不用match



print('实战'+20*'-')
import requests
content = requests.get('https://book.douban.com/').text
print(content)

5.urllib

'''
>>>urllib库
   ---urllib.request   请求模块
   ---urllib.error     异常处理模块
   ---urllib.parse     url解析模块
   ---(urllib.robotparser    robots.txt解析模块) - --非重点
'''

url = 'https://www.baidu.com'

### get请求
import urllib.request
resp = urllib.request.urlopen(url)
print(resp.read().decode('utf-8'))

### post请求
import urllib.parse
import urllib.request
data = bytes(urllib.parse.urlencode({'word':'hello'},encoding='utf8'))
resp = urllib.request.urlopen(url, data=data)
print(resp.read())

### 异常
import urllib.request
import urllib.error
import socket
try:
    resp = urllib.request.urlopen(url, timeout=0.1)
except urllib.error.URLError as e:
    if isinstance(e.reason, socket.timeout):
        print('TIME OUT')

### 响应
resp.status
resp.getheaders()
resp.getheaders('Server')

### 加参数
### Request
import urllib.request
request = urllib.request.Request(url)
resp = urllib.request.urlopen(request)


from urllib import request, parse
headers = {
    'User-Agent': ''}  # request.add_header('','')
data = {'': ''}
req = request.Request(url=url, data=data, headers=headers, method='POST')
resp = request.urlopen(req)

### 代理(handler)
from urllib import request

proxy_handler = request.ProxyHandler({
    'http': '//xxx.x.x.x:xxxx',
    'https':'//xxx.x.x.x: xxxx'
})
opener = request.bulid_opener(proxy_handler)
resp = opener.open(url)

### cookie
import http.cookiejar, urllib.request
cookie = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.bulid_opener(handler)
resp = opener.open(url)
for item in cookie:
    print(item.name + '=' + item.value)


import http.cookiejar,urllib.request
filename ='cookie.txt'
cookie = http.cookiejar.MozillaCookieJar(filename) #http.cookiejar.LWPCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
resp = opener.open(url)
cookie.save(ignore_discard=True,ignore_expires=True)


import http.cookiejar,urllib.request
cookie = http.cookiejar.LWPCookieJar()
cookie.load('cookie.txt',ignore_expires=True,ignore_discard=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
resp = opener.open(url)




# **** 重点 ****
# urlencode
from urllib.parse import urlencode
params = {
    '':'',
    '':''
}
base_url = 'https://www.baidu.com?'
url = base_url + urlencode(params)

# url解析
from urllib.parse import urlparse
result = urlparse(url)  #协议 默认https
result = urlparse(url,scheme='https')
result = urlparse(url,allow_fragments=False)

 

6.xpath

7.bs

from bs4 import BeautifulSoup as bs
url = ''
soup = bs(url,'lxml')
soup.prettify() # 格式化
soup.title.string  # title标签的内容


#>>>>标签选择器
#　选择元素
print(soup.head) # head标签
print(soup.p)   #　p标签
#　获取名称
soup.title.name   # －－－＞title
# 获取属性
soup.p.attrs('name')   #　p标签的name属性
soup.p('name')
# 获取内容
soup.p.string   #　ｐ标签的内容
#　嵌套选择
soup.head.title.string   #　head标签下的title标签的内容
# 子孙节点
soup.p.contents   # ｐ标签的子节点
soup.p.children   # 以迭代器的方式返回子节点
for i,child in enumerate(soup.p.children):
    print(i,child)
soup.p.descendants   #　子孙节点　迭代器
#　父节点／祖先节点
soup.p.parent    # 父节点
soup.p.parents  #　祖先节点
#　兄弟节点
soup.a.next_siblings
soup.a.previous_siblings


#>>>>　标准选择器
# find_all
soup.find_all('li')  #　所有的li标签
soup.find_all('li')[0]  #　第一个li标签
#　attrs
soup.find_all(attrs={'id':'list_a'}) #　id = list_a 的所有标签
soup.find_all(id='list_a') #　id = list_a 的所有标签
soup.find_all(class_='list_a') #　class = list_a 的所有标签
# text
soup.find_all(text='')  # 标签文本内容

#find_next_siblings()
#find_next_sibling()
#find_previous_siblings()
#find_previous_sibling()
#find_next()
#find_previous()
#find_all_next()
#find_all_previous()

#>>>ｃss选择器
#　select
soup.select('.class_ .class_1')
soup.select('ul li')
tar = soup.select('#id .class_') #id＝＇＇下的 class＝＇＇的标签
soup.select('ul')[0]
#　获取属性
tar['']
tar.attrs['']
#　获取内容
tar.get_text()