Python实例 -- 爬虫

#coding="utf-8"

import urllib2
import re
import threading
import time

"""
抓取代理发布页的ip和port10 http://www.xici.net.co/nn/%d
"""

proxylist = []


def get_proxy_from_cnproxy():
    global proxylist
    
    p = re.compile(r'<td><img alt="(.+?)" src=".+?" /></td>[sS]*?<td>(.+?)</td>[sS]*?<td>(.+?)</td>[sS]*?<td>[sS]*?<a href=".+?">.+?</a>[sS]*?</td>[sS]*?<td>.+?</td>[sS]*?<td>(.+?)</td>')
    
    for i in range(1,2):
        target = r"http://www.xici.net.co/nn/%d" %i
        print target
        req = urllib2.urlopen(target)
        result = req.read()
        matchs = p.findall(result)
        for record in matchs:
            addr = record[0]
            ip = record[1]
            port = record[2]
            protocol = record[3]
            l = [ip, port, protocol, addr]
            #print l
            proxylist.append(l)    
        print proxylist


class ProxyCheck(threading.Thread):
    def __init__(self, proxylist, fname):
        threading.Thread.__init__(self)
        self.proxylist = proxylist
        self.timeout = 5
        self.test_url = "http://www.baidu.com/"
        self.test_str = "030173"
        self.checkedPProxyList = []
        self.fname = fname
        
    def checkProxy(self):
        cookies = urllib2.HTTPCookieProcessor()
        for proxy in self.proxylist:
            proxy_handler = urllib2.ProxyHandler({"http":r'http://%s:%s'%(proxy[0],proxy[1])})
            opener = urllib2.build_opener(cookies, proxy_handler)
            opener.addheaders = [('user-agent', 'mozilla/5.0(iphone; u; cpu like mac os x; en) applewebkit/420+ (khtml, like gecko) version/3.0 mobile/1A537a safari/419.3')]
            urllib2.install_opener(opener)
            t1 = time.time()
            try:
                req = urllib2.urlopen(self.test_url, timeout = self.timeout)
                result = req.read()
                timeused = time.time() - t1
                pos = result.find(self.test_str)
                if pos > 1:
                    self.checkedPProxyList.append([proxy[0],proxy[1],proxy[2],proxy[3],timeused])
                else:
                    continue;
            except Exception,e:
                print e.message
                continue;
            
    def sort(self):
        sorted(self.checkedPProxyList,cmp=lambda x,y:cmp(x[4],y[4]))
    
    def save(self):
        f = open(self.fname, 'w+')
        for proxy in self.checkedPProxyList:
            f.write("%s:%s	%s	%s	%s
"%(proxy[0],proxy[1],proxy[2],proxy[3],str(proxy[4])))
        f.close()
        
    def run(self):
        self.checkProxy()
        self.sort()
        self.save()
        

if __name__ == "__main__":
    get_proxy_from_cnproxy()
    t1 = ProxyCheck(proxylist,"test.txt")
    t1.start()