马蜂窝景点数据采集

马蜂窝数据采集，难点sn 加密 采用固定字符串 加上参数转化md5值截取部分md5

import hashlib
import pymongo
import pandas
import requests
import time
from pyquery import PyQuery as pq
from retry import retry


class ScenicSpot:
    def __init__(self):
        self.scenic_url = "http://www.mafengwo.cn/ajax/router.php"
        self.headers = {
            'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
        }
        self.client = pymongo.MongoClient().ScenicSpot.ScenicSpot
        self.pio_headers = {
            'Host': 'www.mafengwo.cn',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
            'Cookie': 'PHPSESSID=o98g37f4squ0aq4ubcr07d84f2; mfw_uuid=5fe69be4-4047-f9a5-fa34-c5d367eb316b; oad_n=a%3A3%3A%7Bs%3A3%3A%22oid%22%3Bi%3A1029%3Bs%3A2%3A%22dm%22%3Bs%3A15%3A%22www.mafengwo.cn%22%3Bs%3A2%3A%22ft%22%3Bs%3A19%3A%222020-12-26+10%3A11%3A48%22%3B%7D; __jsluid_h=8b69bee30f0e6459c08df76385484c05; __omc_chl=; __omc_r=; __mfwc=direct; __mfwa=1608948709480.47993.1.1608948709480.1608948709480; __mfwb=0418cf79c433.1.direct; __mfwlv=1608948709; __mfwvn=1; __mfwlt=1608948709; Hm_lvt_8288b2ed37e5bc9b4c9f7008798d2de0=1608948710; Hm_lpvt_8288b2ed37e5bc9b4c9f7008798d2de0=1608948710; uva=s%3A78%3A%22a%3A3%3A%7Bs%3A2%3A%22lt%22%3Bi%3A1608948710%3Bs%3A10%3A%22last_refer%22%3Bs%3A6%3A%22direct%22%3Bs%3A5%3A%22rhost%22%3Bs%3A0%3A%22%22%3B%7D%22%3B; __mfwurd=a%3A3%3A%7Bs%3A6%3A%22f_time%22%3Bi%3A1608948710%3Bs%3A9%3A%22f_rdomain%22%3Bs%3A0%3A%22%22%3Bs%3A6%3A%22f_host%22%3Bs%3A3%3A%22www%22%3B%7D; __mfwuuid=5fe69be4-4047-f9a5-fa34-c5d367eb316b; bottom_ad_status=1; UM_distinctid=1769cd106714ce-0be3d1620af6d5-3e604809-1fa400-1769cd106729c0; CNZZDATA30065558=cnzz_eid%3D1359122677-1608945050-%26ntime%3D1608945050; __jsl_clearance=1608948735.479|0|LP8kMR7h6lJOyF4aqU9yvnUg4Ek%3D'
            }
        self.all_list = []
        tunnel = "tps198.kdlapi.com:15818"
        username = "t10886694756492"
        password = "bjgfg7jn"
        self.proxies = {
            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
            "https": "https://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
        }

    def par(self, t):
        hl = hashlib.md5()
        hl.update(t)
        return hl.hexdigest()[2:12]

    def get_page(self):
        for i in range(18, 21):
            t = time.time() * 1000
            print('第' + str(i) + '页')
            page = str(i)
            qdata = '{"_ts":"' + str(t) + '","iMddid":"10794","iPage":"' + str(
                page) + '","iTagId":"0","sAct":"KMdd_StructWebAjax|GetPoisByTag"}c9d6618dbc657b41a66eb0af952906f1'
            sn = self.par(qdata.encode('utf-8'))
            data = {
                'sAct': 'KMdd_StructWebAjax|GetPoisByTag',
                'iMddid': '10794',
                '_ts': t,
                'iPage': page,
                'iTagId': '0',
                '_sn': sn
            }
            response = requests.post(url=self.scenic_url, headers=self.headers, data=data, proxies=self.proxies)
            data = response.json()['data']['list']
            doc = pq(data)
            li_list = doc('li').items()
            for li in li_list:
                title = li('a').attr("title")
                title_url = "http://www.mafengwo.cn" + li('a').attr("href")
                self.get_point_info(title_url, title)
                time.sleep(2)

    def get_point_info(self, url, title):
        """
        cookie 有问题
        """
        poi_dict = {}
        poi_dict['景区名称'] = title
        self.pio_headers['Referer'] = url
        print(url)
        poi_resp = requests.get(url, headers=self.pio_headers)
        if poi_resp.status_code == 521:
            raise
        poi_doc = pq(poi_resp.content)
        introduction = poi_doc('.summary').text()
        poi_dict['景区介绍'] = introduction
        place_station = poi_doc('.mod.mod-location p').text()
        dl_list = poi_doc('.mod.mod-detail dl').items()
        for dl in dl_list:
            dt = dl('dt').text()
            if '门票' in dt:
                dd = dl('dd').text()
                poi_dict['门票'] = dd
            elif '开放时间' in dt:
                dd = dl('dd').text()
                poi_dict['开放时间'] = dd
            else:
                continue
        poi_dict['景点位置'] = place_station
        self.client.insert_one(poi_dict)
        print(poi_dict)
        self.all_list.append(poi_dict)

    def run(self):
        self.get_page()
        pandas.DataFrame(self.all_list).to_excel('旅游景点.xlsx', index=False)


if __name__ == '__main__':
    ScenicSpot().run()