爬虫_淘宝（selenium）

总体来说代码还不是太完美

实现了js渲染网页的解析的一种思路

主要是这个下拉操作，不能一下拉到底，数据是在中间加载进来的，

具体过程都有写注释

from selenium import webdriver
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from lxml import etree
import re
import pymongo

client = pymongo.MongoClient('127.0.0.1', port=27017)
db = client.taobao_mark
collection = db.informations


def search(url):

    # 设置无头浏览器
    opt = webdriver.ChromeOptions()
    opt.set_headless()
    driver = webdriver.Chrome(options=opt)
    driver.get(url)

    # 使用显示等待加载输入文本框,设置搜索关键字，以'轻薄本'为例
    input_key = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, 'q')))
    input_key.send_keys('轻薄本')

    # 显示等待加载搜索按钮,然后执行点击操作
    search_btn = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'btn-search')))
    search_btn.click()

    # 搜索出来的网页内容只有一行与关键字相关，剩下的都是广告
    # 此时发现更多相关内容在//span[@class="see-all"]这个标签的链接中，
    # 通过xpath获取链接，进行跳转
    element = etree.HTML(driver.page_source)
    list_page_url = element.xpath('//span[@class="see-all"]/a/@href')[0]
    list_page_url = 'https://s.taobao.com/' + list_page_url
    driver.close()
    return list_page_url
    

def scroll(driver):
    # 如果没有进行动态加载，只能获取四分之一的内容,调用js实现下拉网页进行动态加载
    for y in range(7):
        js='window.scrollBy(0,600)'
        driver.execute_script(js)
        time.sleep(0.5)


def get_detail_url(url):

    # 用来存储列表页中商品的的链接
    detail_urls = []

    opt = webdriver.ChromeOptions()
    opt.set_headless()
    driver = webdriver.Chrome(options=opt)
    # driver = webdriver.Chrome()
    driver.get(url)

    scroll(driver)

    # 通过简单处理，获取当前最大页数
    max_page = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.XPATH, '//div[@class="total"]'))
        )
    text = max_page.get_attribute('textContent').strip()
    max_page = re.findall('d+', text, re.S)[0]    
    # max_page = int(max_page)
    max_page = 1
    # 翻页操作
    for i in range(1, max_page+1):
        print('正在爬取第%s页' % i)
        # 使用显示等待获取页面跳转页数文本框
        next_page_btn = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.XPATH, '//input[@class="input J_Input"]'))
            )

        # 获取确定按钮，点击后可进行翻页
        submit = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.XPATH, '//span[@class="btn J_Submit"]'))
            )

        next_page_btn.clear()
        next_page_btn.send_keys(i)
        # 这里直接点击会报错'提示不可点击'，百度说有一个蒙层，
        # 沉睡两秒等待蒙层消失即可点击
        time.sleep(2)
        submit.click()
        scroll(driver)

        urls = WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located((By.XPATH, '//a[@class="img-a"]'))
        )
        for url in urls:
            detail_urls.append(url.get_attribute('href'))
    driver.close()
    # 返回所有商品链接列表
    return detail_urls

def parse_detail(detail_urls):
    parameters = []
    opt = webdriver.ChromeOptions()
    opt.set_headless()
    driver = webdriver.Chrome(options=opt)
    for url in detail_urls:
        parameter = {}

        print('正在解析网址%s' % url)
        driver.get(url)
        html = driver.page_source
        element = etree.HTML(html)

        # 名字name
        name = element.xpath('//div[@class="spu-title"]/text()')
        name = name[0].strip()
        parameter['name'] = name

        # 价格price
        price = element.xpath('//span[@class="price g_price g_price-highlight"]/strong/text()')[0]
        price = str(price)
        parameter['price'] = price

        # 特点specials
        b = element.xpath('//div[@class="item col"]/text()')
        specials = []
        for i in b:
            if re.match('w', i, re.S):
                specials.append(i)
        parameter['specials'] = specials

        param = element.xpath('//span[@class="param"]/text()')
        # 尺寸
        size = param[0]
        parameter['size'] = size
        # 笔记本CPU
        cpu = param[1]
        parameter['cpu'] = cpu
        # 显卡
        graphics_card = param[2]
        parameter['graphics_card'] = graphics_card
        # 硬盘容量
        hard_disk = param[3]
        parameter['hard_disk'] = hard_disk
        # 处理器主频
        processor = param[4]
        parameter['processor'] = processor
        # 内存容量
        memory = param[5]
        parameter['memory'] = memory
        parameter['url'] = url
        print(parameter)
        print('='*50)
        save_to_mongo(parameter)
        parameters.append(parameter)
    return parameters
        

def save_to_mongo(result):
    try:
        if collection.insert(result):
            print('success save to mongodb', result)
    except Exception:
        print('error to mongo')


def main():
    url = 'https://www.taobao.com/'
    list_page_url = search(url)
    detail_urls = get_detail_url(list_page_url)
    parameters = parse_detail(detail_urls)

if __name__ == '__main__':
    main()

运行结果

 数据库