1 m Queue import Queue 2 from gzip import GzipFile 3 from StringIO import StringIO 4 import time 5 import socket 6 class ContentEncodingProcessor(urllib2.BaseHandler): 7 """A handler to add gzip capabilities to urllib2 requests """ 8 9 # add headers to requests 10 def http_request(self, req): 11 req.add_header("Accept-Encoding", "gzip, deflate") 12 return req 13 14 # decode 15 def http_response(self, req, resp): 16 old_resp = resp 17 18 # if(resp.geturl() != req): 19 # print 'no' 20 # return 1 21 # gzip 22 if resp.headers.get("content-encoding") == "gzip": 23 gz = GzipFile( 24 fileobj=StringIO(resp.read()), 25 mode="r" 26 ) 27 resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) 28 resp.msg = old_resp.msg 29 # deflate 30 if resp.headers.get("content-encoding") == "deflate": 31 gz = StringIO( deflate(resp.read()) ) 32 resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and 33 resp.msg = old_resp.msg 34 return resp 35 36 # deflate support 37 import zlib 38 def deflate(data): # zlib only provides the zlib compress format, not the deflate format; 39 try: # so on top of all there's this workaround: 40 return zlib.decompress(data, -zlib.MAX_WBITS) 41 except zlib.error: 42 return zlib.decompress(data) 43 44 45 #(set timeout) 46 socket.setdefaulttimeout(10) 47 48 encoding_support = ContentEncodingProcessor 49 opener = urllib2.build_opener( encoding_support, urllib2.HTTPHandler) 50 51 class Fetcher: 52 def __init__(self,threads): 53 self.opener = urllib2.build_opener(urllib2.HTTPHandler) 54 self.lock = Lock() #线程锁 55 self.q_req = Queue() #任务队列 56 self.q_ans = Queue() #完成队列import socket 57 self.threads = threads 58 for i in range(threads): 59 t = Thread(target=self.threadget) 60 t.setDaemon(True) 61 t.start() 62 self.running = 0 63 64 def __del__(self): #解构时需等待两个队列完成 65 time.sleep(0.5) 66 self.q_req.join() 67 self.q_ans.join() 68 69 def taskleft(self): 70 return self.q_req.qsize()+self.q_ans.qsize()+self.running 71 72 def push(self,req): 73 self.q_req.put(req) 74 75 def pop(self): 76 return self.q_ans.get() 77 78 def threadget(self): 79 while True: 80 ans = '' 81 req = self.q_req.get() 82 # print req 83 84 with self.lock: #要保证该操作的原子性，进入critical area 85 self.running += 1 86 87 try: 88 # ans = self.opener.open(req).read() 89 #content = opener.open(req).read() 90 content = urllib2.urlopen(req).read() 91 # print temp.geturl() 92 # print req 93 # add gzip support from here 94 ans = str(content) 95 except Exception, what: 96 print what 97 pass 98 99 self.q_ans.put((ans,req)) 100 with self.lock: 101 self.running -= 1 102 self.q_req.task_done() 103 time.sleep(0.01) # don't spam 104 105 if __name__ == "__main__": 106 a = [0] * 3600000 107 links = [ 'http://www.songtaste.com/song/%d/'%i for i in range(1,3600000) ] 108 f = Fetcher(threads=50) 109 for url in links: 110 f.push(url) 111 while f.taskleft(): 112 the_page,x =f.pop() 113 # print the_page 114 try: 115 npos = the_page.index('chart#fav') 116 except : 117 pass 118 else: 119 for j in range(npos,1,-1): 120 if the_page[j] == ',': 121 k = j 122 break 123 sum = 0 ; 124 t = 1 ; 125 for j in range(k-1,1,-1): 126 if the_page[j] <= '9' and the_page[j] >='0': 127 sum = sum + (int(the_page[j]) - int('0')) * t 128 t *= 10; 129 else : 130 break 131 p = int(x[30:-1]) 132 if(p % 10000 <= 5 ) 133 a[p] = sum 134 if sum != 0: 135 print p 136 print sum 137