scrapy 常用代码

一，scrapy请求

yield scrapy.Request(url=url, dont_filter=True, callback=self.page, meta={'item': copy.deepcopy(item)})

yield scrapy.FormRequest(url=self.url, headers=self.unicornHeader, method='POST', formdata=self.FormData, meta=self.customerData, callback=self.after_post, errback=self.error_handle, dont_filter=True)
item = response.meta['item']

二，xpath提取数据

response.xpath('//*[@id="__EVENTVALIDATION"]/@value').extract_first().strip()
response.xpath('//div[@class="mt20 articl-know"][1]/p[4]/span[2]/text()').extract_first().strip()

三，scarpy 判断

if isinstance(item, ArticleViewsCountItem):
if 'food_id' in item.keys():

四，scrapy  登入

start_urls = ['http://renren.com/']

def parse(self, response):
    data = {
        'email': '111',
        'password': 'sssws'
    }
    print('login.....')
    yield scrapy.FormRequest.from_response(response,
                                      formdata=data,
                                      callback=self.next,
                                      )

五.scrapy 启动

scrapy crawl spiderName
scrapy crawl spiderName -s LOG_FILE=spider.log   # 指定日志输出文件

六.设置代理

1.middlewares.py

meta={'item': copy.deepcopy(item), 'proxy': "http://10.133.3.26:1080"}   # 设置单个请求代理  是spider.py中


import requests


class MyproxiesSpiderMiddleware(object):

    def process_request(self, request, spider):
        proxies = requests.get('http://127.0.0.1:5000/get').content.decode('utf-8')
        print(proxies)
        request.meta["proxy"] = "http://{}".format(proxies)
        # request.meta["proxy"] = "http://36.249.49.43:9999"


import logging
import random
import redis
from steam_users.settings import REDIS_HOST, REDIS_POST, REDIS_DATABASE, REDIS_PASSWORD
logger = logging.getLogger(__name__)


class ProxyDownloadMiddleware(object):
    def __init__(self):
        self.conn = redis.Redis(host=REDIS_HOST, port=REDIS_POST, password=REDIS_PASSWORD, db=REDIS_DATABASE)

    def queue_len(self):
        # 获取队列长度
        return self.conn.llen("proxies")

    def get_redis(self):
        # 随机获取redis中的一个ip
        num = random.randint(1, self.queue_len()) - 1
        return self.conn.lindex('proxies', num).decode('utf-8')

    def process_request(self, request, spider):
        if request.url.startswith("http://"):
            request.meta['proxy'] = "http://{proxy_ip}".format(proxy_ip=self.get_redis())
        elif request.url.startswith("https://") and not request.url.startswith('https://steamcommunity'):
            print('ff')
            request.meta['proxy'] = "https://{proxy_ip}".format(proxy_ip=self.get_redis())
        print("using proxy: {}".format(request.meta['proxy']))
        # # 使用私密代理或独享代理需要将用户名和密码进行base64编码，然后赋值给request.headers["Proxy-Authorization"]
        # # 如果是开放代理就不需要以下步骤，直接设置代理IP即可
        # user_password = "{username}:{password}".format(username='username', password='password')
        # b64_user_password = base64.b64encode(user_password.encode("utf-8"))
        # request.headers["Proxy-Authorization"] = "Basic " + b64_user_password.decode("utf-8")
        return None



from xiaoheihe.settings import SCRAPY_PROXIES


class ProxyDownloadMiddleware(object):

    def process_request(self, request, spider):
        if request.url.startswith("http://"):
            request.meta['proxy'] = SCRAPY_PROXIES
        elif request.url.startswith("https://") and not request.url.startswith('https://steamcommunity'):
            request.meta['proxy'] = SCRAPY_PROXIES
        print("using proxy: {}".format(request.meta['proxy']))
        return None

2..setting中开启代理中间件

DOWNLOADER_MIDDLEWARES = {
   'jxy.middlewares.MyproxiesSpiderMiddleware': 543,
}

七.数据入库

import pymysql


class MogujiePipeline(object):
    def __init__(self):
        # 创建数据库连接
        self.db = pymysql.connect(host='localhost', port=3306, database='cfda', user='root', password='root',
                                  charset='utf8')
        # self.db = pymysql.connect(host='115.238.111.198', port=3306, database='spider_yu', user='spider',
        #                           password='Kangce@0608',
        #                           charset='utf8')
        self.cursor = self.db.cursor()

    def process_item(self, item, spider):
        # 判断爬取的字段数据库中是否已经存在
        num = self.cursor.execute('select id from jiankangshuju_food where url="{}"'.format(item["url"]))
        if not num:
            list_key = []
            list_lalues = []
            for key, lalues in item.items():
                list_key.append(key)
                list_lalues.append("'" + str(lalues).replace("'", "‘") + "'")
            # 拼接sql语句
            insert_sql = "insert into jiankangshuju_food({}) values({})".format(', '.join(list_key),
                                                                                     ', '.join(list_lalues))
            try:
                self.cursor.execute(insert_sql)
                self.db.commit()
            except:
                print('insert_sql:', insert_sql)

            # 查询数据
            self.cursor.execute("select * from catalogue")
            data = self.cursor.fetchone()
            data = self.cursor.fetchall()

            # 更新数据
            self.cursor.execute("update catalogue set ''='{}', ''='{}' where id={}".format())
            self.db.commit()

            # 删除数据
            self.cursor.execute("delete from catalogue where id={}".format())
            self.db.commit()

        return item

    def close_spider(self, spider):
        # 关闭数据库的连接
        self.cursor.close()
        self.db.close()　

八.def start_requests(self)

# get请求
def start_requests(self):
    db = pymysql.connect(host='localhost', port=3306, database='game', user='root', password='root',
                         charset='utf8', autocommit=True)
    cursor = db.cursor()
    cursor.execute('select id, appid, last_modified from steam_appid where id =1085660')
    for appid in cursor.fetchall():
        item = {}
        item['appid'] = appid[1]
        item['last_modified'] = appid[2]
        yield scrapy.Request(url='https://store.steampowered.com/app/{}/'.format(appid[1]),
                             meta={'item': copy.deepcopy(item)})


# post  request payload
yield scrapy.Request(url='https://www.wegame.com.cn/api/rail/web/data_filter/game_info/by_game_id',
                     meta={'item': copy.deepcopy(item)},
                     headers={'Content-Type': 'application/json;charset=UTF-8',
                              'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
                              },
                     body=json.dumps({"game_ids": ["{}".format(str(appid[1]))],
                                      "filters": [],
                                      "stamp": {"agent_client_language": "zh_CN"},
                                      "response_format": 0}, ensure_ascii=False),
                     dont_filter=True,
                     method='POST')

九,数据库配置

# ---------服务器mysql配置----------
# MYSQL_HOST = '192.168.107.229'
# MYSQL_POST = 3306
# MYSQL_DATABASE = 'spider_app'
# MYSQL_PASSWORD = '123456'
# MYSQL_USER = 'root'

# -------------本地mysql配置--------------
MYSQL_HOST = '10.133.3.26'
MYSQL_POST = 3306
MYSQL_DATABASE = 'spider_app'
MYSQL_PASSWORD = 'root'
MYSQL_USER = 'root'


from steam_users.settings import MYSQL_HOST, MYSQL_POST, MYSQL_DATABASE, MYSQL_PASSWORD, MYSQL_USER

pymysql.connect(host=MYSQL_HOST, port=MYSQL_POST, database=MYSQL_DATABASE, user=MYSQL_USER,
                             password=MYSQL_PASSWORD,
                             charset='utf8', autocommit=True)