python3爬虫 -----爬取古诗文-------from古诗文网站

# -*- coding: utf-8 -*-
#author:zxy
#Date:2018-10-19


import requests
import re
HEADERS={
    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                 "Chrome/69.0.3497.100 Safari/537.36"
}


def parse_url(url):
    response=requests.get(url,headers=HEADERS)
    text=response.text
    titles=re.findall(r'<divsclass="cont">.*?<b>(.*?)</b>',text,re.DOTALL) #r raw
    dynasties=re.findall(r'<psclass="source">.*?<a.*?>(.*?)</a>',text,re.DOTALL)
    authors=re.findall(r'<psclass="source">.*?<a.*?<a.*?>(.*?)</a>',text,re.DOTALL)
    content_tags=re.findall(r'<divsclass="contson".*?>(.*?)</div>',text,re.DOTALL)
    contents=[]
    for content_tag in content_tags:
        x=re.sub('<.*?>','',content_tag)
        xx=re.sub('。', '。
',x)
        contents.append(xx.strip())
    poems=[]
    for value in zip(titles,dynasties,authors,contents):
        title,dynasty,author,content=value
        poem={
            "title":title,
            "dynasty":dynasty,
            "author":author,
            "content":content
        }
        poems.append(poem)

    with open('poems.txt','w',encoding="utf-8") as f:
        for poem in poems:
            for (key,value) in poem.items():
                if(key=="title"):
                    f.write("{}
".format(value))
                if (key == "dynasty"):
                    f.write("	{}
".format(value))
                if(key=="author"):
                    str="	{}
"
                    f.write(str.format(value))
                if(key=="content"):
                    print(value)
                    f.write("{}


".format(value))
                    # print(x+"{}


".format(value))

if __name__ == '__main__':
    url="https://www.gushiwen.org/default_1.aspx"
    parse_url(url)