python 爬取百度url

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2017-08-29 18:38:23
# @Author  : EnderZhou (zptxwd@gmail.com)
# @Link    : http://www.cnblogs.com/enderzhou/
# @Version : $Id$

import requests
import sys
from Queue import Queue
import threading
from bs4 import BeautifulSoup as bs
import re

# 默认爬取百度76页搜索结果url，调用格式 Python.exe 本文件名称.py 搜索关键字，如关键字含特殊符号使用引号包含起来。
# 爬取结果有txt文档输出。目前尚未能过来百度推广链接，后续有可能会完善。另外后续将会添加同一网站相同路径不通参数url的过滤。
# https://www.baidu.com/s?wd=ichunqiu&pn=10
# wd参数为搜索内容关键字 pn参数控制页码 第二页为10 每页新增10 最大页数参数为750即76页。

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36',}

class BaiduSpider(threading.Thread):
    def __init__(self,queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
             url = self._queue.get()
             try:
                 self.spider(url)
             except Exception as e:
                 # print e
                 pass
        
    def spider(self,url):
        r = requests.get(url=url,headers=headers)
        soup = bs(r.content,'html.parser')
        urllist = soup.find_all(name='a',attrs={'data-click':re.compile(('.')),'class':None,'data-is-main-url':None})
        for i in urllist:
            l = requests.get(url=i['href'],headers=headers)
            if l.status_code == 200:
                ll = l.url.split('/')
                lll = ll[0]+'//'+ll[2]+'
'
                #可根据需求修改是否显示主域名
                sys.stdout.write(lll+l.url+'
')
                f1 = open('out_para.txt','a+')
                f1.write(l.url+'
')
                f1.close()
                with open('out_index.txt') as f:
                    if lll not in f.read():
                        f2 = open('out_index.txt','a+')
                        f2.write(lll)
                        f2.close()

def main(keyword):
    queue = Queue()
    for i in range(0,760,10):
        l = 'https://www.baidu.com/s?wd='+keyword+'&pn='+str(i)
        # print l
        queue.put(l)
    threads = []
    thread_count = 5
    for i in range(thread_count):
        threads.append(BaiduSpider(queue))
    for t in threads:
        t.start()
    for t in threads:
        t.join()

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print 'Enter:python %s keyword' % sys.argv[0]
        sys.exit(-1)
    else:
        f1 = open('out_para.txt','w')
        f1.close()
        f2 = open('out_index.txt','w')
        f2.close()
        main(sys.argv[1])