爬虫_房多多（设置随机数反爬）

本来想正面刚一下这个验证码的，但是一直post不上去，只好设置随机延迟，防止反爬

fangdd.py

import requests
from lxml import etree
import re
from check_code import *    #验证码处理模块
from get_pinyin import *    #汉子转拼音模块
from save_to_mongo import *
import time
import random


class Fangdd():
    def __init__(self):
        user_agent_list = [
            "Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
            "Mozilla/4.0(compatible;MSIE6.0;WindowsNT5.1)",
            "Opera/9.80(WindowsNT6.1;U;en)Presto/2.8.131Version/11.11",
            "Mozilla/5.0(Macintosh;IntelMacOSX10_7_0)AppleWebKit/535.11(KHTML,likeGecko)Chrome/17.0.963.56Safari/535.11",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)"
        ]
        #产生一个随机user-agent
        self.headers={
            #从上面的列表上随机取一个
            "User-Agent":random.choice(user_agent_list)
        }
        # self.c = Code_text()
        # self.rand_time = random.randint(1, 5)

    def get_html(self, url):
        # 请求网页源代码
        response = requests.get(url, headers=self.headers)
        html = response.text
        element = etree.HTML(html)
        # 根据title判断是否反爬
        title = element.xpath('//title/text()')
        title = title[0]
        if title == 'chech captcha':
            print('有内鬼，终止交易(爬虫已被发现，正在进行验证码处理)')
            # self.c.post_code()
            time.sleep(5)
        else:
            return html


    def get_location(self, html):
        # 得到全国各地地名
        addresses = []
        element = etree.HTML(html)
        lis = element.xpath('//div[@class="q3rm0"]/li[position()>1]//a/text()')
        for li in lis:
            # 得到的是汉字，而网址中是拼音，转换一下
            li_pinyin = get_pinyin(li)
            addresses.append(li_pinyin)
        return addresses


    def get_all_url(self, addresses):
        # 对每一个地名的网址进行解析
        urls = []
        for address in addresses:
            addr_url = 'https://%s.fangdd.com/xiaoqu' % address
            urls.append(addr_url)
        return urls
            

    def parse_list_page(self, urls):
        not_found = []  #网址中的拼音不全是汉字转换的，无法统一抓取
        for p_url in urls:
            # 设置随机数进行睡眠，防止反爬
            # time.sleep(self.rand_time)
            html = self.get_html(p_url)
            element = etree.HTML(html)
            # 多音字转换报错，根据title内容来确定
            title = element.xpath('//title/text()')
            title = title[0]
            if title == '很抱歉！您访问的页面不存在！':
                # 因为有的地区名字过长，网址中采用省略写法，所以根据拼音拼接的网址与真实网址不对应,导致404
                print('由于拼音转换与网址不对应，从列表中删除该网址:%s' % p_url)
            else:
                print(title)
                # 当小区数量不足20时,只有一页，最大页数为1
                max_xiaoqu = element.xpath('//p[@class="filter-result"]/b/text()')
                max_xiaoqu = max_xiaoqu[0]
                max_xiaoqu = int(max_xiaoqu)

                if max_xiaoqu <= 20 and max_xiaoqu != 0 :
                    print('该地区只有一页')
                    print('max_xiaoqu=%s' % max_xiaoqu)
                    max_page = 1
                    self.get_informations(p_url, max_page)

                elif max_xiaoqu > 20:
                    # 找到最大页数，来确定循环的边界
                    max_page = element.xpath('//div[@class="pagebox"]/a[position()<last()]/text()')
                    max_page = max_page[-1].strip()
                    max_page = int(max_page)
                    self.get_informations(p_url, max_page)

                # max_xiaoqu == 0的情况
                else:
                    print(p_url+'该地区没有小区')
    

    def get_informations(self, g_url, max_page):
        print('='*20+'正在爬取网址[%s]'%g_url + '='*30)
            
        for pageNo in range(1, max_page+1):
            time.sleep(random.randint(1, 3))
            pageNo_url = g_url+'/?pageNo=%s' % pageNo
            print('='*10 + '正在爬取[%s]=======[%s]页' % (pageNo_url, pageNo) + '='*10 )

            response = requests.get(pageNo_url, headers=self.headers)
            html = response.text
            element = etree.HTML(html)
            lis = element.xpath('//ul[@class="lp-list"]//li')

            for li in lis:
                information = {}
                li_url = li.xpath('.//h3[@class="name"]/a/@href')

                name = li.xpath('.//h3[@class="name"]/a/text()')
                name = name[0]

                address = li.xpath('.//div[@class="house-cont"]//p[@class="address"]/text()')
                address = address[0]
                address = re.sub(' ', '', address)
                price = li.xpath('.//div[@class="fr"]/span/em/text()')
                price = price[0].strip()
                price = re.sub('元/㎡', '', price)
                information = {
                    'url': li_url,
                    'name': name, 
                    'address': address, 
                    'price': price
                }
                save_to_mongo(information)        


def main():
    f = Fangdd()
    url = 'https://shanghai.fangdd.com/'
    html = f.get_html(url)
    address = f.get_location(html)
    urls = f.get_all_url(address)
    f.parse_list_page(urls)
        

if __name__ == '__main__':
    main()

get_pinyin.py

from pypinyin import pinyin


def get_pinyin(text):
    # style=0参数设置取消声调，详情http://pypinyin.mozillazg.com/zh_CN/v0.9.1/
    p = pinyin(text, style=0)
    # [['chong'], ['qing']]
    a = []
    for i in p:
        a.append(i[0])
    # print(p)
    b = ''.join(a)
    return b

save_to_mongo.py

import pymongo

client = pymongo.MongoClient('127.0.0.1', port=27017)
db = client.fangdd_mark
collection = db.informations

def save_to_mongo(result):
    try:
        if collection.insert(result):
            pass
            # print('success save to mongodb')
    except Exception:
        print('error to mongo')
            

因为设置了延迟，再加上数据量比较大，所以爬取时间有点长，我打完了一把王者荣耀，c开头的还没爬完，此时数据库中已经有22000条信息了

运行结果：