1 # -*- coding:utf-8 -*- 2 import requests 3 import json 4 import re 5 import os 6 import gevent 7 import time 8 import random 9 from multiprocessing.dummy import Pool as ThreadPool 10 from bs4 import BeautifulSoup 11 class CommentCrawl(object): 12 ''' 13 用来爬取新浪微博评论数据 14 ''' 15 headers = { 16 'User-Agent': '', 17 'Cookie': ''} 18 ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 19 all_comment = [] 20 def __init__(self,urlll,file_name): 21 self.urlll = urlll 22 self.file_name=file_name 23 def base62_decode(self,string, alphabet=ALPHABET): 24 base = len(alphabet) 25 strlen = len(string) 26 num = 0 27 idx = 0 28 for char in string: 29 power = (strlen - (idx + 1)) 30 num += alphabet.index(char) * (base ** power) 31 idx += 1 32 return num 33 34 def parser_url(self): 35 code = self.urlll.split('?')[0].split('/')[-1] 36 code1 = code[0] 37 code2 = code[1:5] 38 code3 = code[5:] 39 id1 = self.base62_decode(code1) 40 id2 = self.base62_decode(code2) 41 id3 = self.base62_decode(code3) 42 numList = [id1, id2, id3] 43 plus = ''.join(map(str, numList)) 44 comment_url = 'http://weibo.com/aj/v6/comment/big?ajwvr=6&id='+ plus +'&root_comment_max_id_type=0&page={}' 45 return comment_url 46 def get_url_page(self): 47 r = requests.get(self.parser_url().format(1),headers=self.headers) 48 data = json.loads(r.text) 49 total_page = data['data']['page']['totalpage'] 50 return total_page 51 52 def all_urls(self): 53 all_urls = [self.parser_url().format(i + 1) for i in range(self.get_url_page())] 54 return all_urls 55 56 def comment_parser(self,html): 57 soup = BeautifulSoup(html, 'html.parser') 58 data = soup.select('.WB_text') 59 comment = [i.text.split('：')[-1] for i in data] 60 return comment 61 def finnal_text(self,url): 62 finnal_all_comment=''.join(self.all_comment) 63 r1 = requests.get(url,headers=self.headers) 64 time.sleep(random.randint(1,5)) 65 data1 = json.loads(r1.text) 66 html =data1['data']['html'] 67 finnal_data = self.comment_parser(html) 68 self.all_comment+=finnal_data 69 print(len(self.all_comment)) 70 return finnal_all_comment 71 def save_file(self,url): 72 path = os.getcwd() 73 filename = self.file_name + '.txt' 74 file = path + '/' + filename 75 f = open(file, 'a+', encoding='utf-8') 76 f.write(self.finnal_text(url)) 77 78 if __name__ == "__main__": 79 aa = CommentCrawl('http://weibo.com/2202387347/EFdPHe50Z?from=page_1006062202387347_profile&wvr=6&mod=weibotime','小米6发布会') 80 all_link = aa.all_urls() 81 pool=ThreadPool(4) 82 results = pool.map(aa.save_file,all_link) 83 pool.close() 84 pool.join()