爬虫_古诗文网（正则表达式）

程序中请求到的和网页中内容不一样，但也是古诗，不是道是不是因为请求头的原因，使得网站推荐的古诗有差异

import requests
import re

headers = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'

def get_html(url):
    try:
        response = requests.get(url, headers)
        response.raise_for_status()
        response.encoding = response.apparent_encoding
        return response.text
    except:
        print('get_html(url) faild')


def parse_html(html):
    titles = re.findall(r'<div class="cont">.*?<b>(.*?)</b>', html, re.DOTALL)
    dynasties = re.findall(r'<p class="source"><a.*?>(.*?)</a>', html, re.DOTALL)
    authors = re.findall(r'<p class="source"><a.*?><a.*?>(.*?)</a>', html, re.DOTALL)
    content_tags = re.findall(r'<div class="contson".*?>(.*?)</div>', html, re.DOTALL)
    contents = []
    for content in content_tags:
        content = re.sub(r'<.*?>', '', content)
        contents.append(content.strip())
    poems = []    
    for value in zip(titles, dynasties, authors, contents):
        title, dynasties, authors, content = value
        poem = {
            'title': title,
            'dynasties': dynasties,
            'authors': authors,
            'content': content
        }
        print(poem)
        poems.append(poem)
    return poems


def main():
    page_num = 10
    for i in range(1, page_num+1):
        url = 'https://www.gushiwen.org/default_{0}.aspx'.format(i)
        html = get_html(url)
        parse_html(html)


if __name__ == '__main__':
    main()

运行结果