降低耦合性获取微博数据

微博采集

import json
import queue
import re
import threading

import requests
import xlrd
from lxml import etree
from pymongo import MongoClient

proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
            "host": "http-dyn.abuyun.com",
            "port": "9020",
            "user": "H6VZC52B4BF2986D",
            "pass": "7850C72DC876E723",
        }


class WB:
    def __init__(self):
        self.start_temp_url = "https://m.weibo.cn/api/container/getIndex?type=uid&value={}"
        self.headers = {
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"
        }
        self.proxies = {
            "http": proxyMeta,
            "https": proxyMeta,
        }
        self.save_content_q = queue.Queue()
        self.url_q = queue.Queue()
        self.client = MongoClient(host='localhost', port=27017)
        # 使用的mongo的数据表
        self.db = self.client.WbImageSet

    def save_data_mongodb(self, collect_name, data):
        # 定义一个去重的图标集名称
        self.collect_name = self.db[collect_name]
        history_record = self.collect_name.find_one({"_id": data['id']})
        if history_record:
            # 数据库中已经存在数据
            pass
        else:
            # 数据库中不存在，插入数据
            self.collect_name.update_one({'_id': data['id']}, {'$set': data}, upsert=True)

    def get_author_info(self, url):
        """
        获取博主信息
        :return:
        """
        response = requests.get(url=url, headers=self.headers, proxies=self.proxies, verify=True)
        self.data = response.content.decode('utf8')
        content = json.loads(self.data).get('data')
        max_content = content.get('userInfo').get('statuses_count')
        return max_content

    def get_containerid(self):
        """获取微博主页的containerid，爬取微博内容时需要此id"""
        content = json.loads(self.data).get('data')
        for data in content.get('tabsInfo').get('tabs'):
            if (data.get('tab_type') == 'weibo'):
                containerid = data.get('containerid')
        return containerid

    def get_url(self, containerid, max_content):
        for x in range(int(max_content)//10):
            wb_content_url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=' + self.name + '&containerid='+containerid+'&page='+str(x)
            self.url_q.put(wb_content_url)

    def get_wb_content(self):
        """获取微博内容信息,并保存到文本中，内容包括：每条微博的内容、微博详情页面地址、点赞数、评论数、转发数等"""
        num = 0
        while True:
            try:
                if self.url_q.empty():
                    break
                weibo_url = self.url_q.get()
                response = requests.get(url=weibo_url, headers=self.headers, proxies=self.proxies, verify=True)
                content = json.loads(response.content.decode('utf8')).get('data')
                cards = content.get('cards')
                if (len(cards) > 0):
                    for j in range(len(cards)):
                        num += 1
                        card_type = cards[j].get('card_type')
                        item = {}
                        if (card_type == 9):
                            mblog = cards[j].get('mblog')
                            scheme = cards[j].get('scheme')  # 微博地址
                            print(scheme)
                            print("--正在抓取--{}--第{}条微博--".format(self.name, num))
                            text_id = mblog.get("id")  # 微博id 用于去重
                            text = mblog.get('text')  # 微博内容
                            # 以下为获取标题的正则匹配
                            html = etree.HTML(text)
                            x = html.xpath('//text()')
                            title = ','.join(x)
                            title = title.replace('
', '').replace('
', '').replace('	', '')
                            title = re.sub('(#.*#)', '', title)
                            title = re.sub('@', '', title)
                            title = re.sub(' ', '', title)
                            pictures = mblog.get('pics')  # 正文配图，返回list
                            pic_urls = []  # 存储图片url地址
                            if pictures:
                                for picture in pictures:
                                    pic_url = picture.get('large').get('url')
                                    pic_urls.append(pic_url)
                            if pic_urls == []:
                                continue
                            item['id'] = text_id
                            item['category'] = self.category
                            item['author'] = self.name
                            item['title'] = title
                            item['url'] = pic_urls
                            item['select'] = 0
                            # 去重逻辑
                            self.save_data_mongodb(self.name, item)
            except Exception as e:
                print(e)

    def run(self):
        while True:
            if wb_content_q.empty():
                break
            dict_wb = wb_content_q.get()
            self.category = dict_wb['category']
            self.name = dict_wb['name']
            self.wb_id = dict_wb['id']
            max_content = self.get_author_info(self.start_temp_url.format(self.wb_id))
            containerid = self.get_containerid()
            self.get_url(containerid, max_content)

            Threads_caiji = []
            for x in range(5):
                t1 = threading.Thread(target=self.get_wb_content)
                Threads_caiji.append(t1)
            for a in Threads_caiji:
                a.start()
                a.join()


class Excel_path:

    @staticmethod
    def get_excel_info(path,num):
        """
        从excel中读取数据 所有链接切割把博主的id分割出来，按照 id:category 的方式保存为字典
        :param path:  excel路径
        :return:返回从excel中读取的字典
        """
        excel_sheet = xlrd.open_workbook(path).sheet_by_index(num)
        category_name = excel_sheet.col_values(0)[1:]
        wb_url = excel_sheet.col_values(2)[1:]
        name = excel_sheet.col_values(1)[1:]
        for i in wb_url:
            item = {}
            a = wb_url.index(i)
            c = i.split('?')[0]
            d = c.split('/')[-1]
            item['name'] = name[a]
            item['category'] = category_name[a]
            item['id'] = d
            print(item)
            wb_content_q.put(item)


if __name__ == '__main__':
    wb_content_q = queue.Queue()
    excel_path = 'D:\gongsi_code\ImageSpider\微博\Image_set\数据源.xlsx'
    excel_index = 0
    Excel_path.get_excel_info(excel_path, int(excel_index))
    WB().run()

采集微博数据 将数据存入mongo 所有链接存mongo中 

下载器

import os
import queue
import re
import threading
import time
from concurrent.futures.thread import ThreadPoolExecutor

import pymongo
import requests
import xlrd


class WbDownload:
    def __init__(self):
        self.client = pymongo.MongoClient(host='localhost', port=27017)
        self.db = self.client.WbImageSet
        self.info_q = queue.Queue()

    def get_info(self, collection_name):
        self.collection = self.db[collection_name]
        a = self.collection.find({"select": 0})
        for index in a:
            self.info_q.put(index)

    def save_(self):
        while True:
            """保存到本地"""
            if self.info_q.empty():
                break
            image = self.info_q.get()
            category_name = image['category']
            upload_time = time.strftime("%Y-%m-%d", time.localtime())
            rule = re.compile(r's*', re.S)
            rule2 = re.compile(r'W*', re.S)
            title = rule.sub('', image['title'])
            title = rule2.sub('', title)
            path = 'D:/微博/' + category_name + '/' + str(upload_time) + '/' + title
            if os.path.exists(path):
                continue
            else:
                os.makedirs(path)
            with open(path + '/content.txt', 'w', encoding='utf8')as fb:
                fb.write(str([image['title']]))
            for x in image['url']:
                x_index = image['url'].index(x)
                if x[-4:-1:] == 'gif':
                    continue
                with open(path + '/{}.jpg'.format(str(x_index)), 'wb') as f:
                    response = requests.get(url=x)
                    f.write(response.content)
            self.collection.update_one({"id": id}, {"$set": {"select": 1}})
            print('-----  ' + title + '  写入完成  ------')


    def run(self):
        while True:
            if name_q.empty():
                break
            name = name_q.get()
            self.get_info(name)
            Threads = []
            for i in range(20):
                t_down = threading.Thread(target=self.save_)
                t_down.start()
                Threads.append(t_down)
            for t in Threads:
                t.join()




class Excel_path:

    @staticmethod
    def get_excel_info(path,num):
        """
        从excel中读取数据 所有链接切割把博主的id分割出来，按照 id:category 的方式保存为字典
        :param path:  excel路径
        :return:返回从excel中读取的字典
        """
        excel_sheet = xlrd.open_workbook(path).sheet_by_index(num)
        name = excel_sheet.col_values(1)[1:]
        for x in name:
            print(x)
            name_q.put(x)


if __name__ == '__main__':
    name_q = queue.Queue()
    path = 'D:\gongsi_code\ImageSpider\微博\Image_set\数据源.xlsx'
    excel_index = 0
    Excel_path.get_excel_info(path, excel_index)
    with ThreadPoolExecutor(5) as executor:
            wb = WbDownload()
            executor.submit(wb.run)

专门从mongo 中取数据然后下载，下载完后修改mongo中字段的名称，避免重复使用数据