<?php namespace AppConsoleCommandsMining; use AppConsoleCommandsCoreBaseCommand; use AppRepositoryCoreScriptStatusRepo; use AppRepositoryMiningMoversTopKRepo; use AppRepositoryMiningMoversTopKMinersRepo; use IlluminateSupportFacadesDB; use AppCoreCommonHelpersMiningHelper; /** * Listing分析脚本 * @auth ives * @since 20170606 0.0.1 */ class GetTopKAsins extends BaseCommand { /** * The name and signature of the console command. * DATA_CENTER_MANAGE_SYSTEM * @var string */ protected $signature = 'movers {country}'; /** * The console command description. * * @var string */ protected $description = '获取爬虫采集的飙升榜 参数:国家码'; /** * @var */ public $country; /** * @var */ public $createDatas; /** * @var ScriptStatusRepo */ public $statHandler; /** * @var 暂时略过的 listid>1000 的数据,后续分析 */ public $listidThan1000Arr = []; /** * Create a new command instance. * * @return void */ //當前爬蟲飆升榜最大主鍵 public $moversTopKRepoMaxId; public function __construct( MoversTopKRepo $moversTopKRepo , MoversTopKMinersRepo $moversTopKMinersRepo ) { parent::__construct(); $this->moversTopKRepo = $moversTopKRepo; $this->moversTopKMinersRepo = $moversTopKMinersRepo; } public function handle() { $this->info('start 获取爬虫采集的飙升榜'); $country = $this->argument('country'); $countrys = MiningHelper::getCountrys(); if (!in_array($country, $countrys)) { $this->error($country . ' is not country code'); exit(); } $this->country = $country; //任务名称 //支持多国家同时运行 $this->statHandler = new ScriptStatusRepo('GetTopKAsins_' . $country . ''); //开启任务 $endPoint = $this->start(); //TODO 需结合脚本自身和爬虫脚本执行频率修改 //从panew.amazon_keyword_* 表中取 1000 条记录 分块处理 //一次選3000 $step = 3000; $id_range = $this->moversTopKRepoMaxId - $step; $db_getMoversTopK = $this->moversTopKRepo->getMoversTopK($id_range); foreach ($db_getMoversTopK as $k => $asinArr) { $this->mainFun($asinArr); } //结束任务 $this->finish(); } public function start() { //任务已存在则不执行 if (!$this->statHandler->scriptExist()) { $this->info('任务已存在则不执行script exist..'); exit(); } $statHandler = $this->statHandler; $endPoint = $statHandler->getEndPoint(); $this->info('上次執行后,分析至amazon_movers_' . $this->country . '的主鍵=' . $endPoint); $maxId = $this->moversTopKRepo->getMoversTopKMaxId(); $maxId = $maxId[0]['maxId']; $this->moversTopKRepoMaxId = $maxId; if ($maxId <= $endPoint) { $this->info('當前amazon_movers_' . $this->country . '最大主鍵--maxId=' . $maxId . '無新數據待分析'); exit(); } else { $this->info('已經校驗當前amazon_movers_' . $this->country . '最大主鍵--maxId=' . $maxId . '---開始分析---'); } //开启 $statHandler->scriptStart($endPoint); return $endPoint; } public function finish() { $statHandler = $this->statHandler; $statHandler->freshEndPoint($this->moversTopKRepoMaxId); $statHandler->scriptEnd(2); $this->error('更新脚本狀態表endPoint=' . $this->moversTopKRepoMaxId . ':本次完成分許,正常結束--script finished!'); } public function mainFun($asinArr) { foreach ($asinArr as $k => $v) { $$k = $v; } //sales_rank_now > 1000 pass if ($asinArr['sales_rank_now'] > 1000) { $this->info('sales_rank_now=' . $asinArr['sales_rank_now'] . '---排名大于1000'); return; } $if_existed = $this->moversTopKMinersRepo->chkAsinExisted($asinArr['asin']); if ($if_existed) { $this->info('miner_movers_shakers_' . $this->country . '中存在asin=' . $asinArr['asin'] . ':跳过'); return; } else { $this->info('miner_movers_shakers_' . $this->country . '中不存在asin=' . $asinArr['asin'] . ':继续'); } $data['asin'] = $asin; $data['img'] = $pic_url; $data['first_inrank'] = $add_time; $data['first_rapidly'] = $sales_rank_ago; $data['created_at'] = date('Y-m-d H:i:s'); $data['updated_at'] = date('Y-m-d'); $data['price'] = $price; $data['bsr1'] = $sales_rank_now; $data['bsr1path'] = $cate_id; $data['reviews'] = $review_counts; // 首次 $data['status'] = 1; $data['is_top1000'] = 1; $data['day_span'] = 14; $data['top_count'] = 1; $data['is_steady'] = 0; $data['end_date'] = date("Y-m-d", strtotime("+14 Days")); $data_add_quote = array(); foreach ($data as $k => $v) { $data_add_quote[$k] = '`' . $v . '`'; } // $data = $data_add_quote; $table = 'miner_movers_shakers_' . $this->country; try { DB::connection('mining_db') ->table($table) ->insert($data); $this->info('对' . $table . '新增asin=' . $asin); } catch (Exception $exception) { $this->info('db异常停止:' . $exception); $this->statHandler->scriptEnd(3); } //对listing采集表collect_listing_asins以asin+country为条件添加或者更新 $table_collect_listing_asins = 'collect_listing_asins'; try { $data = array(); $data['asin'] = $asin; $data['country'] = $this->country; $res = DB::connection('mining_db') ->table($table_collect_listing_asins) ->select('id', 'pass_at') ->where('country', '=', $this->country) ->where('asin', '=', $asin) ->where('status', '=', 1) ->orderBy('id', 'desc') ->limit(1) ->first(); $str_info = $table_collect_listing_asins . 'asin=' . $asin . '+country=' . $this->country . '+sataus=1--'; if (empty($res)) { $this->info('查询' . $str_info . '无数据' . ':新增'); $data['is_fresh'] = 1; $data['status'] = 1; $data['updated_at'] = date('Y-m-d H:i:s'); $data['pass_at'] = date("Y-m-d", strtotime("+14 Days")); //TODO DB::connection('mining_db') ->table($table_collect_listing_asins) ->insert($data); } else { $id = $res->id; $pass_at = $res->pass_at; $this->info('查询' . $str_info . '有数据' . ':检查是否更新'); $diff = strtotime($pass_at) - strtotime("+14 Days"); $data_update = array(); $data_update['pass_at'] = date("Y-m-d", strtotime("+14 Days")); if ($diff < 0) { $this->info('----需更新'); DB::connection('mining_db') ->table($table_collect_listing_asins) ->where('id', '=', $id) ->update($data_update); } else { $this->info('----不需更新'); } } } catch (Exception $exception) { $this->info('db异常停止:' . $exception); $this->statHandler->scriptEnd(3); } } }
$diff = strtotime($pass_at) - strtotime("+14 Days"); $data_update = array(); $data_update['pass_at'] = date("Y-m-d", strtotime("+14 Days"));