Python爬取代理ip

# -*- coding:utf-8 -*-
#author : willowj
import urllib
import urllib2
from bs4 import BeautifulSoup
import re
import bs4

import sys


reload(sys)  
sys.setdefaultencoding('utf8') 


def ip_test(ip,url= "https://www.baidu.com"):
    #test ip if can be used
    #url = "http://ip.chinaz.com/getip.aspx"# 默认测试网址
    ip1="http://"+ip
    try :
        res = urllib.urlopen(url,proxies={'http:':ip1}).read() #尝试代理访问
        print 'ok',ip1 #,res
        return True
    except Exception,e:
        print "failed"
        return False
    

def get_iphtml_inyoudaili():
    url='http://www.youdaili.net'
    html=urllib2.urlopen(url) 
    code=html.read()
    #href="http://www.youdaili.net/Daili/http/26672.html" title="12月27号 最新代理http服务器ip地址"
    regexp='href="(.*?)" .*?最新代理http服务器ip地址'
    pat=re.compile(regexp)
    met=re.findall(pat,code)
    print met[0]
    #最新代理http服务器ip地址 html
    return met[0]
    

def getIps(url):
    #getip from website, test,and  return,save aviable ips in 'ips.txt'
    htmlip=urllib2.urlopen(url) 
    codeip=htmlip.read()

    regexpip='([1-9][0-9]{0,2}.S*?)@HTTP#'  #IP样式
    pat_ip=re.compile(regexpip) 

    met_ip=re.findall(pat_ip,codeip)

    ips=[]
    file_open=open('ips.txt','w')
    for x in met_ip:
        print x
        if ip_test(x):
            ips.append(x)
            file_open.write(x+'
')
    file_open.close()
    #print ips,'youdaili'
    return ips


def saveIps(list):
    file_open=open('ips.txt','w')
    for ip in list:
        file_open.write(ip+'
')
    file_open.close()


def read_ips(file='ips.txt'):
    '''读取IP 以list返回'''
    file_open=open(file)
    lines=file_open.readlines()
    ips=[]
    for line in  lines:
        ip=line.strip("
")
        ips.append(ip)
    print ips
    return ips
 
 
if __name__=="__main__":
    
    ips = getIps(get_iphtml_inyoudaili())

    saveIps(ips)