doraemon的python 爬虫示例1（入门）动态数据的爬取

数据来源某某总局

import requests
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
fp = open('./yao.txt','w',encoding='utf-8')
# fp = open('./company_detail.txt','w',encoding='utf-8')

for page in range(1,2):
    url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList'
    data = {
        'on': 'true',
        'page': str(page),
        'pageSize': '15',
        'productName': '',
        'conditionType': '1',
        'applyname': '',
        'applysn': '',
    }
# 就是访问这个网站的时候，所需要携带的参数，英文是post请求，所以是data
    
#         data = {
#         'on': 'true',
#         'page': str(page),
#         'pageSize': '15',
#         'productName': '',
#         'conditionType': '1',
#         'applyname': '',
#         'applysn': '',
#     }

#     data_dic = requests.post(url=url,data=data,headers=headers).json()
    data_dic = requests.post(url=url,data=data,headers=headers).json()
#     print(data_dic)
    for dic in data_dic['list']:
        _id = dic['ID']
        post_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById'
        post_data={'id':_id}
        detail_dic = requests.post(url=post_url,data=post_data,headers=headers).json()
               
        company_title = detail_dic['epsName']
        address = detail_dic['epsProductAddress']

        fp.write(company_title+':'+address+'
')
        print(company_title,'爬取成功!!!')               
                                 
fp.close() 

总结：如果那里出错，当前语句没有错误，那么最好向上看代码