1 import requests 2 from requests.exceptions import RequestException 3 from pyquery import PyQuery as pq 4 from bs4 import BeautifulSoup 5 import pymongo 6 from config import * 7 from multiprocessing import Pool 8 import time 9 10 client = pymongo.MongoClient(MONGO_URL) # 申明连接对象 11 db = client[MONGO_DB] # 申明数据库 12 13 def get_one_page_html(url): # 获取网站每一页的html 14 headers = { 15 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " 16 "Chrome/85.0.4183.121 Safari/537.36" 17 } 18 try: 19 response = requests.get(url, headers=headers) 20 if response.status_code == 200: 21 return response.text 22 else: 23 return None 24 except RequestException: 25 return None 26 27 28 def get_room_url(html): # 获取当前页面上所有room_info的url 29 soup = BeautifulSoup(html, 'lxml') 30 addresses = soup.find_all('p', {'class': 'content__list--item--des'}) 31 doc = pq(html) 32 room_urls = doc('.content__list--item--main .content__list--item--title a').items() 33 return room_urls, addresses 34 35 36 def parser_room_page(room_html, address_queue1, address_queue2, address_queue3): # 对租房详情页面进行解析，获取特定信息 37 soup = BeautifulSoup(room_html, 'lxml') 38 pinpai = soup.find('p', {'class': 'content__aside__list--subtitle oneline'}).text.strip().split(' ')[0] 39 price = soup.find_all('li', {'class': 'table_col'}) 40 zujin = price[6].text # 租金 41 yajin = price[7].text # 押金 42 fuwufei = price[8].text # 服务费 43 zhongjiefei = price[9].text # 中介费 44 house_type = soup.find('ul', {'class': 'content__aside__list'}).find_all('li')[1].text[5:11] # 户型 45 x = soup.find_all('li', {'class': 'fl oneline'}) 46 area = x[1].text[3:] # 面积 47 floor = x[7].text[3:] # 楼层 48 direction = x[2].text[3:] # 朝向 49 elevator = x[8].text[3:] # 有无电梯 50 carport = x[10].text[3:] # 有无车位 51 tenancy = x[18].text[3:] # 租期 52 maintenance = x[4].text[3:] # 维护日期 53 kanfang = x[21].text[3:] # 看房是否要预约 54 tags = soup.find('p', {'class': 'content__aside--tags'}).get_text().replace(' ', '') # 标签 55 56 yield { 57 'pinpai': pinpai, 58 'zujin': zujin, 59 'yajin': yajin, 60 'fuwufei': fuwufei, 61 'zhongjiefei': zhongjiefei, 62 'house_type': house_type, 63 'area': area, 64 'floor': floor, 65 'direction': direction, 66 'elevator': elevator, 67 'carport': carport, 68 'tenancy': tenancy, 69 'maintenance': maintenance, 70 'kanfang': kanfang, 71 'location1': address_queue1.pop(), 72 'location2': address_queue2.pop(), 73 'location3': address_queue3.pop(), 74 'tags': tags, 75 } 76 77 78 def save_to_mongo(result): 79 if db[MONGO_TABLE].insert_one(result): 80 print('存储到mongodb成功', result) 81 return True 82 return False 83 84 85 def main(page): 86 url = 'http://sz.xxxxx.com/zufang/pg' + str(page) + 'rt200600000002/#contentList' 87 html = get_one_page_html(url) 88 room_urls, addresses = get_room_url(html) 89 address_queue1 = [] # 采用队列数据结构，先进先出，用来存放租房区域（南山区、福田区等） 90 address_queue2 = [] 91 address_queue3 = [] # 采用队列数据结构，先进先出，用来存放租房具体小区 92 for address in addresses: 93 temp = address.find_all('a') 94 address_queue1.insert(0, temp[0].text) 95 address_queue2.insert(0, temp[1].text) 96 address_queue3.insert(0, temp[2].text) 97 for room_url in room_urls: 98 room_url_href = room_url.attr('href') 99 room_url_href = 'http://sz.xxxxx.com/' + room_url_href 100 room_html = get_one_page_html(room_url_href) 101 if room_html is None: # 非常重要，否则room_html为None时会报错 102 pass 103 else: 104 # parser_room_page(room_html, address_queue1, address_queue2, address_queue3) 105 results = parser_room_page(room_html, address_queue1, address_queue2, address_queue3) 106 for result in results: 107 save_to_mongo(result) 108 109 if __name__ == '__main__': 110 time1 = time.time() 111 pool = Pool() # 使用多进程提高爬取效率 112 pool.map(main, [i for i in range(1, 101)]) 113 time2 = time.time() 114 print(time2 - time1) # 耗时