使用python获取房价信息

从贝壳网获取房价信息。

基本的步骤和我的这篇博文一样：https://www.cnblogs.com/mrlayfolk/p/12319414.html。不熟悉的可参考一下。

下面的代码是获取3000个样本的代码。

# encoding:utf-8

'''
目的：从贝壳找房中爬取房价信息。网址：https://cd.ke.com/ershoufang/qingyang/l2/
环境：python 3.7.3
所需的库：requests、BeautifulSoup、xlwt
'''

import logging
import xlwt
import requests
import string
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
    "Host": "cd.ke.com",
}

# 将获取的信息保存到表格中
def save_info(content):
    workbook = xlwt.Workbook(encoding = 'ascii')
    worksheet = workbook.add_sheet('house info')
    style = xlwt.XFStyle() # 初始化样式
    font = xlwt.Font() # 为样式创建字体
    font.name = 'Times New Roman' 
    font.bold = True # 黑体
    font.underline = True # 下划线
    font.italic = True # 斜体字
    style.font = font # 设定样式
    worksheet.write(0, 0, '名称')
    worksheet.write(0, 1, '位置')
    worksheet.write(0, 2, '房屋信息')
    worksheet.write(0, 3, '总价（万）')
    worksheet.write(0, 4, '单价（元/平方米）')
    
    for i, item in enumerate(content):
        for j in range(5):  #多添加一列（序号）
            worksheet.write(i+1, j, content[i][j])
    workbook.save('./house_info.xls') # 保存文件


# 获取房屋相关的信息
# 主要包括：title positon houseinfo totalprice unitprice
def get_info():
    all_info        = []
    title_list      = []
    position_list   = []
    house_list      = []
    totalPrice_list = []
    unitPrice_list  = []
    
    for i in range(100):
        link = 'https://cd.ke.com/ershoufang/qingyang/pg%dl2/' % i
        r = requests.get(link, headers=headers, timeout=10)
        print (str(i+1), 'status_code: ', r.status_code)
        soup = BeautifulSoup(r.text, 'lxml')
        titleInfo = soup.findAll('div', {'class': 'info clear'})
        positionInfo = soup.findAll('div', {'class': 'positionInfo'})
        houseInfo = soup.findAll('div', {'class': 'houseInfo'})
        totalPrice = soup.findAll('div', {'class': 'totalPrice'})
        unitPrice = soup.findAll('div', {'class': 'unitPrice'})
        for item in titleInfo:
            title = item.div.a.text.strip()
            title_list.append(title)
        for item in positionInfo:
            postion = item.a.text.strip()
            position_list.append(postion)
        for item in houseInfo:
            house = item.text.strip().replace('
', ' ').replace(' ', '')
            house_list.append(house)
        for item in totalPrice:
            total_price = item.span.text.strip()
            totalPrice_list.append(total_price)
        for item in unitPrice:
            unit_price = item.span.text.strip().replace('单价', '').replace('元/平米', '')
            unitPrice_list.append(unit_price)
    print (len(title_list))
    print (len(position_list))
    print (len(house_list))
    print (len(totalPrice_list))
    print (len(unitPrice_list))
    for i in range(len(title_list)):
        item = [title_list[i], position_list[i], house_list[i], totalPrice_list[i], unitPrice_list[i]]
        all_info.append(item)

    return all_info


if __name__ == "__main__":
    all_info = get_info()
    save_info(all_info)