简单的python爬虫 爬的乌云漏洞类型

import urllib.request
import re

starturl="http://wooyun.org/searchbug.php?q=%E9%87%91%E8%9E%8D"

def get_html_response(url):
    html_response = urllib.request.urlopen(url).read().decode('utf-8')
    return html_response

def geturl(starturl):
    a=get_html_response(starturl)
    childurl=(re.findall(r'/bugs/wooyun-w*-w*',a))
    return childurl

def get_nextpage(starturl):
    d=get_html_response(starturl)
    nextpage=(re.findall(r'searchbug.php?q=金融&pNO=w',d))
    return nextpage

result=[]
result2=[]

for i in get_nextpage(starturl):
    result+=geturl('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))
    #扫描各种漏洞的url地址放入result中
result=set(result)#去除result中重复的地址
#print(len(result))
#测试结果为87，正确

#信息存储+文件处理
output=open("D:\a.txt",'w+')

for i in result:
    k=get_html_response('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))#下载页面到k，可以把
    info=re.findall(r'<h3>w*：s*w*s*</h3>',k)#空白字符用/s,寻找所有适用于<h3>标签的文字
    for j in info:
        j=re.sub(r'：s',':',j)
        j=re.sub(r'<h3>','',j)
        j=re.sub(r'</h3>',',',j)
        j=j.split()
        output.writelines(j)
        output.writelines("
")
output.close()

mentor要求的，简略分析乌云关于金融方面的漏洞并归类

人生第一个能跑的有意义的程序，好开心

2014/8/14根据导师要求做了修改

import urllib.request
import re

starturl="http://www.wooyun.org/searchbug.php?q=6YeR6J6N"

def get_html_response(url):
    html_response = urllib.request.urlopen(url).read().decode('utf-8')
    return html_response

def geturl(starturl):
    a=get_html_response(starturl)
    childurl=(re.findall(r'/bugs/wooyun-w*-w*',a))
    return childurl

def get_nextpage(starturl):
    d=get_html_response(starturl)
    nextpage=(re.findall(r'searchbug.php?q=6YeR6J6N&pNO=w',d))
    return nextpage

result=[]for i in get_nextpage(starturl):
    result+=geturl('http://wooyun.org/'+re.sub('金融','6YeR6J6N',i))
    #扫描各种漏洞的url地址放入result中
result=set(result)#去除result中重复的地址
#print(len(result))
#测试结果为87，正确

#信息存储+文件处理
output=open("D:\wooyun.txt",'w+')

for i in result:
    k=get_html_response('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))#下载页面到k
    info=re.findall(r'<h3>w*：s*w*s*</h3>',k)#空白字符用/s,寻找所有适用于<h3>标签的文字
    detail=re.findall(r'<p class="detail">.*.</p>',k)
    for j in info:
        j=re.sub(r'：s',':',j)
        j=re.sub(r'<h3>','',j)
        j=re.sub(r'</h3>','',j)
        j=j.split()
        output.writelines(j)
        output.writelines("
")
    for j in detail:
        j=re.sub(r'：s',':',j)
        j=re.sub(r'<psclass="detail">','',j)
        j=re.sub(r'</p>','',j)
        j=re.sub(r'"starget="_blank"><imgssrc="/upload/.*.width="600"/></a>','',j)
        j=re.sub(r'<a href="',' http://www.wooyun.org',j)
        j=j.split()
        output.writelines(j)
        output.writelines("
")
    output.writelines("
")
output.close()