随笔写一个简单的爬虫

目标：爬取damai网上即将上演的信息

#!/usr/bin/python
# -*- coding: utf-8 -*-

import requests, re
from bs4 import BeautifulSoup

DOWNLOAD_URL = "http://www.damai.cn/bj/"

#获取url页面内容
def download_page(url):
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                        'Chrome/51.0.2704.63 Safari/537.36'}
    data = requests.get(url, headers = headers).content
    return data
#解析html 
def x_page(url):
    soup = BeautifulSoup(url, 'html.parser')
    li_lst = soup.find('div', class_='index-con').next_sibling.next_sibling.find_all('li')
    titles = [i.find('dt').find('a').string for i in li_lst]
    prices = map(lambda x: x.find('p',class_='price').find('strong').text if x.find('p',class_='price').find('strong') is not None else 'none',li_lst)
    time = [i.find('p',class_='time').string for i in li_lst]
    places = [i.find('p',class_='place').string for i in li_lst]
    return titles,prices,time,places

if __name__ == '__main__':
    url = download_page(DOWNLOAD_URL)
    titles, prices, time, places = x_page(url)
    info_lst = zip(titles,prices,time,places)
    #写入文件
    with open('damai.txt','w+') as f:
        for j in info_lst:
            f.write(' '.join(j)+'

')