Python 爬取SeeBug poc

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2017-08-24 21:42:52
# @Author  : EnderZhou (zptxwd@gmail.com)
# @Link    : http://www.cnblogs.com/enderzhou/
# @Version : $Id$

import requests
from bs4 import BeautifulSoup as bs
import threading
import Queue
import urllib
# import os
import time

main_url ='https://www.seebug.org/vuldb/vulnerabilities?category=&order_time=1&order_rank=1&has_all=default&has_vm=default&submitTime=all&has_affect=default&has_poc=true&has_detail=default&level=all&page='#+pagenumber

bug_url = 'https://www.seebug.org/vuldb/ssvid-'#96358

dl_url = 'https://www.seebug.org/vuldb/downloadPoc/'#96358


#SeeBug防爬机制教强，若仅修改cookie无效，请替换全部header头信息
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
'AlexaToolbar-ALX_NS_PH':'AlexaToolbar/alx-4.0.1',
'Cache-Control':'max-age=0',
'Connection':'keep-alive',
'Cookie':'请填写自己的cookie',
'Referer':'https://www.seebug.org/vuldb/vulnerabilities',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36',
}

class SeeBugPoc(threading.Thread):
    def __init__(self,queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
            url = self._queue.get_nowait()
            self.spidet(url)

    def spidet(self,url):
        r = requests.get(url=url,headers=headers)
        soup = bs(r.content,'html.parser')
        vullist = soup.find_all(name='a',attrs={'class':'vul-title'})
        for u in vullist:
            name = u['href'].split('/')[-1]
            l = dl_url+u['href'].split('-')[-1]
            print name + '----' + l
            d = requests.get(url=l,headers=headers)
            f =  open(name+'.txt','w') 
            f.write(d.content)
            f.close
            time.sleep(7.5)
#SeeBug防爬机制教强，此处设置延时7.5秒。后续看心情是否更新添加代理功能

def main():
    queue = Queue.Queue()
    for i in range(1,2185):
        queue.put(main_url+str(i))

    threads = []
    thread_count = 1

    for i in range(thread_count):
        threads.append(SeeBugPoc(queue))

    for t in threads:
        t.start()

    for t in threads:
        t.join()

if __name__ == '__main__':
    main()