Python爬虫之爬取小幻HTTP 代理 ip

网上大多数搜索到的帖子都是西插,快代理的 ip,唯独没有获取小幻的,本着学习的态度,对小幻的代理 ip 列表进行获取.

直接放代码:

import requests
import time
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

ua = UserAgent()

headers = {'User-Agent': ua.ff}

# 获取全国的url 地址
init_url = 'https://ip.ihuan.me/address/5Lit5Zu9.html'

page_tag = 'pagination'
tbody_tag = 'tbody'

init_data = []
page_data = []
REQ_TIMEOUT = 5
check_url = "http://httpbin.org/headers"


def get_init(base_url, current_page=''):
    response = requests.get(base_url, headers=headers)
    print(response)
    result = BeautifulSoup(response.text, 'lxml')
    tables = result.findAll('table')
    tab = tables[0]
    for tr in tab.tbody.findAll('tr'):
        for index, td in enumerate(tr.findAll('td')):
            if index == 0:
                ip = td.find('a').text
                continue

            if index == 1:
                port = td.getText()
                continue

            if index == 4:
                support_https = td.getText()
                continue

            if index == 5:
                support_post = td.getText()
                continue

            if index == 7:
                speed = td.getText().replace('秒', '')
                continue

        init_data.append([ip, port, support_https, support_post, speed])

    pages = result.find_all('ul', class_=page_tag)
    for index, i in enumerate(pages[0].find_all('li')):
        if index == 0:
            continue
        if current_page != i.find('a')['href'] and i.find('a')['href'] not in page_data:
            page_data.append(i.find('a')['href'])
    print(page_data)


def check_content_and_ip(init_data):
    for i in init_data:
        if i[3] == '支持':
            print(i[0], i[1])
            if i[2] == '支持':
                w = 'https://{0}:{1}'.format(i[0], i[1])
                proxies = {'https': w}
            else:
                w = 'http://{0}:{1}'.format(i[0], i[1])
                proxies = {'http': w}

            OrigionalIP = requests.get(check_url, timeout=REQ_TIMEOUT).content
            print('OrigionalIP', OrigionalIP)
            try:
                MaskedIP = requests.get(check_url, timeout=REQ_TIMEOUT, proxies=proxies).content
                print('MaskedIP', MaskedIP)
                if OrigionalIP != MaskedIP:
                    ip_pool.writelines(w + '
')
                    ip_pool.flush()
            except:
                continue

if __name__ == '__main__':
    ip_pool = open('ip_pool.txt', 'w')
    get_init(init_url)

    num = 30   # 循环获取 29 次,数字可修改,加上上面的初始化操作,一共30次
    for _k in range(1, num):
        print(_k, page_data[_k])
        url = init_url + page_data[_k]
        try:
            get_init(url, current_page=page_data[_k])
        except Exception as e:
            print(str(e))

        # page_data = list(set(page_data))
        if _k > num:
            break
        time.sleep(1)

    check_content_and_ip(init_data)
    ip_pool.close()

对爬虫的代理 ip 很感兴趣,就抓取了一下,小试牛刀.
想知道运行的结果,直接复制代码,运行就可以了 
运行环境: python 3.7