• 14days laravel


    <?php
    
    namespace AppConsoleCommandsMining;
    
    use AppConsoleCommandsCoreBaseCommand;
    use AppRepositoryCoreScriptStatusRepo;
    use AppRepositoryMiningMoversTopKRepo;
    use AppRepositoryMiningMoversTopKMinersRepo;
    
    use IlluminateSupportFacadesDB;
    use AppCoreCommonHelpersMiningHelper;
    
    /**
     * Listing分析脚本
     * @auth ives
     * @since 20170606 0.0.1
     */
    class GetTopKAsins extends BaseCommand
    {
        /**
         * The name and signature of the console command.
         * DATA_CENTER_MANAGE_SYSTEM
         * @var string
         */
        protected $signature = 'movers {country}';
    
        /**
         * The console command description.
         *
         * @var string
         */
        protected $description = '获取爬虫采集的飙升榜 参数:国家码';
    
        /**
         * @var
         */
        public $country;
    
        /**
         * @var
         */
        public $createDatas;
    
        /**
         * @var ScriptStatusRepo
         */
        public $statHandler;
    
        /**
         * @var 暂时略过的 listid>1000 的数据,后续分析
         */
        public $listidThan1000Arr = [];
    
        /**
         * Create a new command instance.
         *
         * @return void
         */
    
        //當前爬蟲飆升榜最大主鍵
        public $moversTopKRepoMaxId;
    
    
        public function __construct(
            MoversTopKRepo $moversTopKRepo
            , MoversTopKMinersRepo $moversTopKMinersRepo
        )
        {
            parent::__construct();
            $this->moversTopKRepo = $moversTopKRepo;
            $this->moversTopKMinersRepo = $moversTopKMinersRepo;
        }
    
        public function handle()
        {
            $this->info('start 获取爬虫采集的飙升榜');
    
            $country = $this->argument('country');
            $countrys = MiningHelper::getCountrys();
            if (!in_array($country, $countrys)) {
                $this->error($country . ' is not country code');
                exit();
            }
            $this->country = $country;
            //任务名称
            //支持多国家同时运行
            $this->statHandler = new ScriptStatusRepo('GetTopKAsins_' . $country . '');
    
            //开启任务
            $endPoint = $this->start();
    
            //TODO 需结合脚本自身和爬虫脚本执行频率修改
            //从panew.amazon_keyword_* 表中取 1000 条记录 分块处理
            //一次選3000
            $step = 3000;
            $id_range = $this->moversTopKRepoMaxId - $step;
            $db_getMoversTopK = $this->moversTopKRepo->getMoversTopK($id_range);
            foreach ($db_getMoversTopK as $k => $asinArr) {
                $this->mainFun($asinArr);
            }
    
            //结束任务
            $this->finish();
        }
    
        public function start()
        {
            //任务已存在则不执行
            if (!$this->statHandler->scriptExist()) {
                $this->info('任务已存在则不执行script exist..');
                exit();
            }
            $statHandler = $this->statHandler;
            $endPoint = $statHandler->getEndPoint();
    
            $this->info('上次執行后,分析至amazon_movers_' . $this->country . '的主鍵=' . $endPoint);
            $maxId = $this->moversTopKRepo->getMoversTopKMaxId();
            $maxId = $maxId[0]['maxId'];
            $this->moversTopKRepoMaxId = $maxId;
            if ($maxId <= $endPoint) {
                $this->info('當前amazon_movers_' . $this->country . '最大主鍵--maxId=' . $maxId . '無新數據待分析');
                exit();
            } else {
                $this->info('已經校驗當前amazon_movers_' . $this->country . '最大主鍵--maxId=' . $maxId . '---開始分析---');
    
            }
    
            //开启
            $statHandler->scriptStart($endPoint);
            return $endPoint;
    
        }
    
        public function finish()
        {
            $statHandler = $this->statHandler;
            $statHandler->freshEndPoint($this->moversTopKRepoMaxId);
            $statHandler->scriptEnd(2);
            $this->error('更新脚本狀態表endPoint=' . $this->moversTopKRepoMaxId . ':本次完成分許,正常結束--script finished!');
        }
    
    
        public function mainFun($asinArr)
        {
    
            foreach ($asinArr as $k => $v) {
                $$k = $v;
            }
    
            //sales_rank_now > 1000 pass
            if ($asinArr['sales_rank_now'] > 1000) {
                $this->info('sales_rank_now=' . $asinArr['sales_rank_now'] . '---排名大于1000');
                return;
            }
    
            $if_existed = $this->moversTopKMinersRepo->chkAsinExisted($asinArr['asin']);
            if ($if_existed) {
                $this->info('miner_movers_shakers_' . $this->country . '中存在asin=' . $asinArr['asin'] . ':跳过');
                return;
            } else {
                $this->info('miner_movers_shakers_' . $this->country . '中不存在asin=' . $asinArr['asin'] . ':继续');
    
            }
    
            $data['asin'] = $asin;
            $data['img'] = $pic_url;
            $data['first_inrank'] = $add_time;
            $data['first_rapidly'] = $sales_rank_ago;
            $data['created_at'] = date('Y-m-d H:i:s');
            $data['updated_at'] = date('Y-m-d');
            $data['price'] = $price;
            $data['bsr1'] = $sales_rank_now;
            $data['bsr1path'] = $cate_id;
            $data['reviews'] = $review_counts;
            // 首次
            $data['status'] = 1;
            $data['is_top1000'] = 1;
            $data['day_span'] = 14;
            $data['top_count'] = 1;
            $data['is_steady'] = 0;
    
            $data['end_date'] = date("Y-m-d", strtotime("+14 Days"));
            $data_add_quote = array();
            foreach ($data as $k => $v) {
                $data_add_quote[$k] = '`' . $v . '`';
            }
            // $data = $data_add_quote;
            $table = 'miner_movers_shakers_' . $this->country;
            try {
    
                DB::connection('mining_db')
                    ->table($table)
                    ->insert($data);
                $this->info('对' . $table . '新增asin=' . $asin);
            } catch (Exception $exception) {
                $this->info('db异常停止:' . $exception);
                $this->statHandler->scriptEnd(3);
            }
    
    
            //对listing采集表collect_listing_asins以asin+country为条件添加或者更新
            $table_collect_listing_asins = 'collect_listing_asins';
            try {
                $data = array();
                $data['asin'] = $asin;
                $data['country'] = $this->country;
    
                $res = DB::connection('mining_db')
                    ->table($table_collect_listing_asins)
                    ->select('id', 'pass_at')
                    ->where('country', '=', $this->country)
                    ->where('asin', '=', $asin)
                    ->where('status', '=', 1)
                    ->orderBy('id', 'desc')
                    ->limit(1)
                    ->first();
                $str_info = $table_collect_listing_asins . 'asin=' . $asin . '+country=' . $this->country . '+sataus=1--';
                if (empty($res)) {
                    $this->info('查询' . $str_info . '无数据' . ':新增');
                    $data['is_fresh'] = 1;
                    $data['status'] = 1;
                    $data['updated_at'] = date('Y-m-d H:i:s');
                    $data['pass_at'] = date("Y-m-d", strtotime("+14 Days"));
                    //TODO
                    DB::connection('mining_db')
                        ->table($table_collect_listing_asins)
                        ->insert($data);
                } else {
    
                    $id = $res->id;
                    $pass_at = $res->pass_at;
    
                    $this->info('查询' . $str_info . '有数据' . ':检查是否更新');
                    $diff = strtotime($pass_at) - strtotime("+14 Days");
                    $data_update = array();
                    $data_update['pass_at'] = date("Y-m-d", strtotime("+14 Days"));
    
                    if ($diff < 0) {
                        $this->info('----需更新');
                        DB::connection('mining_db')
                            ->table($table_collect_listing_asins)
                            ->where('id', '=', $id)
                            ->update($data_update);
                    } else {
                        $this->info('----不需更新');
                    }
                }
    
            } catch (Exception $exception) {
                $this->info('db异常停止:' . $exception);
                $this->statHandler->scriptEnd(3);
            }
    
    
        }
    
    }
                    $diff = strtotime($pass_at) - strtotime("+14 Days");
                    $data_update = array();
                    $data_update['pass_at'] = date("Y-m-d", strtotime("+14 Days"));
  • 相关阅读:
    使用 Selenium
    Senium 简介
    第8章 动态渲染页面爬取
    Ajax 结果提取
    Ajax 分析方法
    WINDOW 2008多人访问设置
    Windows 2012设置允许单个用户连接多个会话的方法
    Windows Server 2012开启多人远程
    BOM 表
    修复材料工单分配材料订单重复占料问题的开发
  • 原文地址:https://www.cnblogs.com/rsapaper/p/7020519.html
Copyright © 2020-2023  润新知