python爬虫练习--爬取所有微博

Ajax，全称为Asynchronous JavaScript and XML，即异步的JavaScript和XML。它不是一门编程语言，而是利用JavaScript在保证页面不被刷新、页面链接不改变的情况下与服务器交换数据并更新部分网页的技术。

对于传统的网页，如果想更新其内容，那么必须要刷新整个页面，但有了Ajax，便可以在页面不被全部刷新的情况下更新其内容。在这个过程中，页面实际上是在后台与服务器进行了数据交互，获取到数据之后，再利用JavaScript改变网页，这样网页内容就会更新了。

可以到W3School上体验几个示例来感受一下：http://www.w3school.com.cn/ajax/ajax_xmlhttprequest_send.asp

代码如下：

#! /usr/bin/env python
# coding: utf-8

import requests
from pyquery import PyQuery as pq
import pymysql
from pymongo import MongoClient
import time

'''
抓取所有微博信息，是通过Ajax进行分析抓取
'''


headers = {
    'Host': 'm.weibo.cn',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
    'Referer': 'https://m.weibo.cn/u/2830678474'
}

def get_page(page):
    '''
    请求url，并获取内容
    :param page:
    :return:
    '''
    params = {
        'type':'uid',
        'value':'2830678474',
        'containerid':'1076032830678474',
        'page':page

    }
    url = 'https://m.weibo.cn/api/container/getIndex'
    try:
        res = requests.get(url,headers=headers,params=params)
        if res.status_code == 200:
            return res.json()
    except requests.ConnectionError as e:
        print('Error',e.args)



def parse_page(json,page):
    if json:
        items = json.get('data').get('cards')
        for index, item in enumerate(items):
            if page == 1 and index == 1:
                continue
            else:
                item = item.get('mblog')
                weibo = {}
                weibo['id'] = item.get('id')
                weibo['text'] = pq(item.get('text')).text()
                weibo['attitudes'] = item.get('attitudes_count')
                weibo['comments'] = item.get('comments_count')
                weibo['reposts'] = item.get('reposts_count')
                yield weibo


#往MySQL中存储
def save_mysql(result):

    table = 'weibo'
    keys = ', '.join(result.keys())
    values = ', '.join(['%s'] * len(result))
    db = pymysql.connect(host='127.0.0.1', user='root', password='123456', port=3306,db='spiders',charset="utf8")
    cursor = db.cursor()
    sql = 'INSERT INTO {table}({keys}) VALUES ({values})'.format(table=table, keys=keys, values=values)
    try:
        if cursor.execute(sql, tuple(result.values())):
            print('Successful')
            db.commit()
    except Exception as e:
        print('Failed',e.args)
        db.rollback()
    db.close()


#往mangoDB中存储
def save_to_mongo(result):
    client = MongoClient()
    db = client['weibo']
    collection = db['weibo']
    if collection.insert(result):
        print('Saved to Mongo')



if __name__ == '__main__':
    for page in range(1,11):
        json = get_page(page)
        results = parse_page(json,page)
        for result in results:
            print(result)
            save_mysql(result)
            time.sleep(1)