012 Python 爬虫项目1

# Python 爬虫项目1
　　● Python 网页请求
　　　　requests
　　　　　　POST
　　　　　　GET

　　　　网页状态码

# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import requests

url = "http://www.baidu.com"
unknow = requests.get(url)
print(type(unknow))
print(unknow)

　　　　

　　　　通过标签匹配内容

# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import requests

url = "http://zz.ganji.com/fang1/"
r = requests.get(url)
soup = BeautifulSoup(r.text,'lxml')
for item in soup.find_all('dd'):
    if item['class'] == ['dd-item','title']:
        #print(item)
        print(item.a.string)
        print("----------------------------------------------------")

　　　　通过浏览器复制 copy selector

　　

# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import requests

url = "http://zz.ganji.com/fang1/"
r = requests.get(url)
soup = BeautifulSoup(r.text,'lxml')

#价格获取
title = soup.select('dl > dd.dd-item.info > div.price > span.num')
print(title)
    
title2 = soup.select('dl > dd.dd-item.size > span.first.js-huxing')
print(title2)

title = soup.select('dl > dd.dd-item.info > div.price > span.num')
print(title)
print(type(title[0]))

　　title 的类型还是 标签 Tag

　　　　soup.body.div.div.a 方式获取

# -*- coding: UTF-8 -*-
from bs4 import BeautifulSoup
import requests

url = "http://zz.ganji.com/fang1/"
r = requests.get(url)
soup = BeautifulSoup(r.text,'lxml')
print(soup.body.div.div.a)

from bs4 import BeautifulSoup
import requests

def isdiv(tag):
    return tag.name == 'div'

url = "http://zz.ganji.com/fang1/"
r = requests.get(url)
soup = BeautifulSoup(r.text,'lxml')

value = soup.find_all(isdiv)
print(value)

　　　　python 使用代理发送网页请求

import requests   
proxies = { "http": "http://10.10.1.10:3128", "https": "http://10.10.1.10:1080", }   
requests.get("http://example.org", proxies=proxies)