# _*_ coding: utf-8 _*_ """ 思路: 1.列表页使用phantomjs模拟点击2.每个链接只抓取第一页9-10条内容,按照标题去重 3.布置定时任务,每天8点执行一次 """ import MySQLdb import redis import sys import os import re import urllib import requests import time import hashlib import traceback import urlparse import random import signal # import multiprocessing import matplotlib matplotlib.use("Agg") import shutil import socket #图片下载延迟的 socket.setdefaulttimeout(30) import multiprocessing from config import IConfig from video_list import ydzx_url_list from bs4 import BeautifulSoup from upload_images import UploadFile from moviepy.editor import VideoFileClip from selenium import webdriver from selenium.webdriver.common.desired_capabilities import DesiredCapabilities reload(sys) sys.setdefaultencoding('utf-8') class WxpnVideo(multiprocessing.Process): def __init__(self): self.redisConf = IConfig.load('resource.redis') self.redisServer = redis.Redis(host=self.redisConf['host'], port=self.redisConf['port'], db=self.redisConf['db'], password=self.redisConf['passwd']) self.dbConfig = IConfig.load('resource.mysql') self.conn = MySQLdb.connect( user = self.dbConfig['user'], passwd = self.dbConfig['password'], db = self.dbConfig['dbname'], host = self.dbConfig['host'], charset = "utf8", use_unicode = True) self.conn.ping(True) self.cursor = self.conn.cursor() self.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36', 'Host': 'www.yidianzixun.com', 'X-Requested-With': 'XMLHttpRequest', 'Connection': 'keep-alive', 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', } self.domain = IConfig.load('resource.domain') self.apiConf = IConfig.load('resource.apiurl') self.key_video_list = 'wxpn:video:list' self.key_title = 'wxpn:video:title' self.storeConfig = IConfig.load('resource.store') self.thumb_path = self.storeConfig['images_path'] self.ossConf = IConfig.load('resource.oss') self.key_id = self.ossConf['access_key_id'] self.key_secret = self.ossConf['access_key_secret'] self.endponit = self.ossConf['endponit'] self.img_upload = UploadFile() self.auth = self.img_upload.auth_oss(self.key_id, self.key_secret) self.videoConf = IConfig.load('resource.apiurl') self.video_publish = self.videoConf['video_publish_api'] self.ydzx_page_api = self.videoConf['ydzx_page_api'] self.start_time = int(time.time()) multiprocessing.Process.__init__(self) def store_video_list_redis(self, video_list): if video_list: for per_list in video_list: if not self.redisServer.sismember(self.key_video_list, per_list): self.redisServer.sadd(self.key_video_list, per_list) else: return False def get_video_para(self): while True: if self.redisServer.scard(self.key_video_list) == 0: break link = self.redisServer.spop(self.key_video_list) print(link) # url = self.ydzx_page_api + link # try: # res = requests.get(url=url, timeout=60) # except Exception as e: # print('连接失败') # print(res.status_code) dcap = dict(DesiredCapabilities.PHANTOMJS) dcap["phantomjs.page.settings.userAgent"] = ( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36" ) try: driver = webdriver.PhantomJS(desired_capabilities=dcap, executable_path='/usr/local/phantomjs/bin/phantomjs') # driver.set_page_load_timeout(10) # driver.set_script_timeout(10) time.sleep(random.randrange(3, 8)) driver.get(link) time.sleep(random.randrange(2, 6)) text = driver.page_source driver.service.process.send_signal(signal.SIGTERM) driver.quit() except Exception as e: print(traceback.format_exc()) continue # if res.status_code == 200: soup = BeautifulSoup(text, 'lxml') title_list = soup.select('div.channel-news div.doc-title') itemid_list = soup.select('div.channel-news a.style-content-middle') if title_list and itemid_list: try: for num, title in enumerate(title_list): m = hashlib.md5() m.update(str(title.text).strip()) psw = m.hexdigest() print(title.text) itemid = itemid_list[num]['data-docid'] if not self.redisServer.sismember(self.key_title, psw): yield psw, itemid except Exception as e: print(traceback.format_exc()) continue else: print('一点资讯视频列表{0}页链接请求失败,请及时查看原因'.format(link)) # self.form_data['content'] = '【创业黑马】预警:一点资讯视频列表{0}页链接请求失败,请及时查看原因'.format(link) # res = requests.post(self.msg_api, data=self.form_data) def time_cycle(self,origin_time): now = time.time() try: if origin_time == '昨天': published = int(now) - 24*3600 elif '天' in origin_time: day_one = re.compile('(.*?)天') published = int(now)-int(day_one.findall(origin_time)[0])*24*3600 elif '小时' in origin_time: hour_one = re.compile('(.*?)小时') published = int(now)-int(hour_one.findall(origin_time)[0])*24*60 elif '分' in origin_time: min_one = re.compile('(.*?)分') published = int(now)-int(min_one.findall(origin_time)[0])*60 elif '月' in origin_time: month_one = re.compile('(.*?)个月') published = int(now)-int(month_one.findall(origin_time)[0])*30*24*3600 else: timeArray = time.strptime(origin_time, "%Y.%m.%d") published = int(time.mktime(timeArray)) return published except Exception as e: print(traceback.format_exc()) def download_video(self, psw, itemid): now = int(time.time()) url = 'http://www.yidianzixun.com/article/' + itemid print(url) self.headers['Referer'] = url try: res = requests.get(url=url, headers=self.headers, timeout=60) print(res.status_code) except Exception as e: print('小链接连接失败') if res.status_code == 200: soup = BeautifulSoup(res.text, 'lxml') title = soup.select('div.left-wrapper > h2')[0].text try: video_src = soup.select('div.video-wrapper > video')[0]['src'] except Exception as e: print('此篇为文章,不是视频') thumb_src = soup.select('div.video-wrapper > video')[0]['poster'] try: source = soup.select('body.page-article .left-wrapper > .meta > a')[0].text except Exception as e: source = soup.select('body.page-article .left-wrapper > .meta > span')[0].text source_re = re.sub('来源：', '', str(source)) source = source_re publishtime = soup.select('body.page-article .left-wrapper > .meta > span')[0].text try: timestamp = self.time_cycle(str(publishtime)) except Exception as e: timestamp = now img_url_parts = urlparse.urlparse(thumb_src) img_url_query = urlparse.parse_qs(img_url_parts.query,True) if img_url_query.has_key('wx_fmt'): ext_name = '.' + img_url_query['wx_fmt'][0] else: ext_name = '.png' thumb_p = self.thumb_path + 'video/thumb' if not os.path.exists(thumb_p): os.mkdir(thumb_p) img_down_local_path = thumb_p + '/' + psw[:20] + ext_name urllib.urlretrieve(thumb_src, img_down_local_path) file_name = psw[:20] + ext_name if os.path.exists(img_down_local_path): images_path = self.ossConf['video_thumb_path'] status = self.img_upload.upload_to_oss(self.auth, self.endponit, images_path, file_name, img_down_local_path) thumb_src = self.domain['img_url_oss'] + 'Cmstop/ydzx/' + file_name m = hashlib.md5() m.update(str(thumb_src)) psw_thumb = m.hexdigest() try: delay_re = re.compile('"duration":(d+)') playtime = delay_re.findall(str(res.text))[0] except Exception as e: print(traceback.format_exc()) playtime = None video_path = self.thumb_path + 'video/' + str(video_src).split('/')[-1] video_res = requests.get(video_src).content with open(video_path, 'wb') as f: f.write(video_res) try: clip = VideoFileClip(video_path) print(clip.duration) except Exception as e: print(traceback.format_exc()) return False # with open(video_path, 'r') as f: # length = len(f.read()) # if length < 819200: # return False video_name = str(video_src).split('/')[-1][10:] if os.path.exists(video_path): images_path = self.ossConf['video_path'] status = self.img_upload.upload_to_oss(self.auth, self.endponit, images_path, video_name, video_path) print('') if status != 'success': return False print('视频上传成功') video_link = self.domain['img_url_oss'] + 'Cmstop/video/ydzx/' + video_name try: sql = "insert into cmstop_comment_topic(title, description, thumb, created, url_md5, url) values(%s, '', %s, %s, %s, '')" params = (title, thumb_src, now, psw_thumb) self.cursor.execute(sql, params) self.conn.commit() topicid = self.cursor.lastrowid except Exception as e: print(traceback.format_exc()) self.conn.rollback() sourceid = self.get_article_sourceid(source) try: result = self.cursor.execute(""" insert into cmstop_content(topicid, sourceid, catid, modelid, title, subtitle, source_title, source_link, weight, status, created, score, published, thumb, createdby) values(%s, %s, %s, %s, %s, %s, %s, %s, 60, %s, %s, %s, %s, %s, %s) """, (topicid, sourceid, 47, 4, title, None, title, '', 3, now, 0, timestamp, thumb_src, 0)) self.conn.commit() lastrowid = self.cursor.lastrowid except Exception as e: print(traceback.format_exc()) self.conn.rollback() video_id = str(video_src).split('/')[-1][10:-4] try: sql = "insert into cmstop_video(contentid, video, playtime, author, video_id, aid) values(%s, %s, %s, %s, %s, %s)" print(sql) params = (lastrowid, video_link, playtime, source, video_id, 0) self.cursor.execute(sql, params) self.conn.commit() except Exception as e: print(traceback.format_exc()) self.conn.rollback() self.redisServer.sadd(self.key_title, psw) api_url = self.video_publish + str(lastrowid) try: resp = urllib.urlopen(api_url) result = resp.read() except: print 'connect failed' else: print('一点资讯视频主链接请求失败,请及时查看原因') # self.form_data['content'] = '【创业黑马】预警:一点资讯视频主链接请求失败,请及时查看原因' # res = requests.post(self.msg_api, data=self.form_data) def get_article_sourceid(self, source, medias = []): source = source.strip() sourceid = 0 """ print source print set([source.encode('utf-8')]) print medias """ result = self.cursor.execute('select `sourceid`, `name`, `has_signed_contract` from `cmstop_source` where `name`="' + source + '"') has_signed_contract = 0 if medias and (set([source.encode('utf-8')]) & medias): has_signed_contract = 1 if result: data = self.cursor.fetchone() sourceid = data[0] if data[2] != has_signed_contract: try: result = self.cursor.execute(""" update `cmstop_source` set `has_signed_contract`=%s where sourceid=%s """, (has_signed_contract, sourceid)) self.conn.commit() except: self.conn.rollback() else: try: result = self.cursor.execute(""" insert into `cmstop_source`(`name`, `logo`, `url`, `initial`, `has_signed_contract`) values(%s, %s, %s, %s, %s) """, (source, '', '', '', has_signed_contract)) self.conn.commit() sourceid = self.cursor.lastrowid except: self.conn.rollback() return sourceid def run(self): os.system('pkill phantomjs') lockConf = IConfig.load('resource.lock') lock_file = lockConf['lock_path_ydzx'] if os.path.exists(lock_file): print('lock file exists') return False os.system(r'touch %s '% lock_file) self.store_video_list_redis(ydzx_url_list) get_video_para = self.get_video_para() for psw, itemid in get_video_para: print(psw) stop_time = int(time.time()) balance_time = stop_time - self.start_time if balance_time >= 10800: #运行时间为3个小时 self.del_file(self.thumb_path + 'video') os.system(r'rm -rf %s' % lock_file) os._exit(0) try: self.download_video(psw=psw, itemid=itemid) time.sleep(random.uniform(2, 8)) os.system('pkill ffmpeg-osx-v3.2.4') except Exception as e: print(traceback.format_exc()) continue self.del_file(self.thumb_path + 'video') os.system(r'rm -rf %s' % lock_file) def video_publish(self): sql = 'select contentid from cmstop_video where contentid<=3528920 and contentid>=3430851' self.cursor.execute(sql) data = self.cursor.fetchall() for num in data: api_url = self.video_publish + str(num[0]) try: resp = urllib.urlopen(api_url) result = resp.read() except: print 'connect failed' def del_file(self, path): os.chdir(path) #进入要清空的目录 ds = list(os.listdir(path)) #获得该目录下所有文件或文件夹列表皮 for d in ds: #遍历该列表 if os.path.isfile(d): #如果列表项是文件 os.remove(d) #直接删除 else: #如果不会文件 shutil.rmtree(d) #也直接删除 if __name__ == '__main__': video_one = WxpnVideo() # video_two = WxpnVideo() video_one.start() # video_two.start() video_one.join() # video_two.join()