• Python之免费随机代理IP的获取以及使用


    from bs4 import BeautifulSoup
    import requests
    import random
    #随机代理获取的网站
    url = 'http://www.xicidaili.com/nn/'
    headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
    }

    #将获取到的代理IP存到ip_list中并返回列表
    def get_ip_list(url, headers):
    web_data = requests.get(url, headers=headers)
    soup = BeautifulSoup(web_data.text, 'lxml')
    ips = soup.find_all('tr')
    ip_list = []
    for i in range(1, len(ips)):
    ip_info = ips[i]
    tds = ip_info.find_all('td')
    ip_list.append(tds[1].text + ':' + tds[2].text)
    return ip_list

    #从代理IP列表中随机取出一个IP并返回
    def get_random_ip(ip_list):
    proxy_list = []
    for ip in ip_list:
    proxy_list.append('http://' + ip)
    proxy_ip = random.choice(proxy_list)
    proxies = {'http': proxy_ip}
    return proxies


    if __name__ == '__main__':
    ip_list = get_ip_list(url, headers=headers)
    print(ip_list)
    proxies = get_random_ip(ip_list)
    print(proxies)
       #代理IP的使用
    headers = {
    'User-Agent':'Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3 like Mac OS X; wo-SN) AppleWebKit/535.16.1 (KHTML, like Gecko) Version/4.0.5 Mobile/8B114 Safari/6535.16.1'
    }
    resp = requests.get('https://www.baidu.com',proxies = proxies,headers = headers)
    print(resp.text)
  • 相关阅读:
    Java学习开篇
    《我的姐姐》
    世上本无事,庸人自扰之
    这48小时
    补觉
    淡定
    es java api 设置index mapping 报错 mapping source must be pairs of fieldnames and properties definition.
    java mongodb groupby分组查询
    linux 常用命令
    mongodb too many users are authenticated
  • 原文地址:https://www.cnblogs.com/yp19970/p/12410781.html
Copyright © 2020-2023  润新知