python 使用selenium爬取进击的巨人漫画

import requests
from bs4 import BeautifulSoup
import os
from selenium import webdriver
from selenium.webdriver.firefox.webdriver import WebDriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
class Down_Cartoon():
    def __init__(self):
        self.content_url='https://www.manhuabei.com/manhua/jinjidejuren/'
        self.base_url='https://www.manhuabei.com'
        self.header={"Use-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36" }
        self.html_path=r'd:进击的巨人.txt'
        self.file_path=r'D:OneDrive漫画进击的巨人'
        
    def get_url(self,url):
        '''
        通用url请求
        '''
        r=requests.get(url,headers=self.header)
        if r.status_code==200:
            return r.text
        else:
            return ""           

    def parse_html(self,html_content):
        '''
        BeautifulSoup解析网页
        返回每个章节名称列表和每个章节首页列表
        '''
        soup=BeautifulSoup(html_content,'lxml')
        #self.save_webxml(self.html_path,soup.prettify())
        main=soup.find('ul',class_="list_con_li autoHeight")
        content=main.find_all('a')
        print("总章节:",len(content))
        chapter_url=[]
        title_name=[]
        for p in content:
            title_name.append(p['title'])
            chapter_url.append(p['href'])
        return chapter_url,title_name


    def save_webxml(self,file_path, xml_content):
        '''
        保存html至本地
        '''
        with open(file_path,'w',encoding='UTF-8',errors='ignore') as write_blog:
            write_blog.write(xml_content)

    def download_one_page(self,href,dir_path,num):
        '''
        下载一个图片并保存
        '''
        strpic=str(num+1)+'.jpg'
        full_path=os.path.join(dir_path,strpic)
        if not os.path.exists(full_path):
            try:
                r=requests.get(href,headers=self.header)
                if r.status_code==200:
                     with open(full_path,'wb') as img:
                        img.write(r.content)
                        print(strpic,"success")                 
                else:
                    print(full_path,'下载失败',href)
            except:
                print('下载失败',href)
        else:
            print(strpic,'图片已存在,无需下载')  

    def mkdir(self,own_dir_name):
        '''创建文件夹'''
        own_dir_name=own_dir_name.strip()
        full_path= os.path.join(self.file_path,own_dir_name)
        isExists=os.path.exists(full_path)
        if not isExists:
            #print("创建",own_dir_name,"文件夹")
            os.makedirs(full_path)
            os.chdir(full_path)
            return full_path
        else:
            #print(own_dir_name,'文件夹已存在')
            return full_path

    def run(self):
        content_list,title_list= self.parse_html(self.get_url(self.content_url))
        brower=webdriver.Chrome()
        self.download_content(brower,content_list,title_list)
        brower.quit()                

    def download_content(self,browerdrive,content_list,title_list):
        '''
        下载漫画
        '''
        cartoon_href_list=[]
        for i,title in enumerate(title_list):
            chapter_name=title.split(" ")[0]
            print("正在下载第%s,总共%s话"%(chapter_name,len(title_list)))
            dir_path=self.mkdir(chapter_name)
            full_url=self.base_url+content_list[i]
            browerdrive.get(full_url)
            img_url_list=[]
            chapter_info={}
            try:
                img_info= browerdrive.find_element_by_class_name("img_info")
            except:
                print("爬取失败!")
                continue
            tag_string=img_info.text
            try:
                init_page=browerdrive.find_element_by_css_selector("img[style='display: inline;']").get_attribute('src')
            except:
                print("爬取失败!")
                continue         
            img_url_list.append(init_page)
            num=int(tag_string.split('/')[1][0:2])
            print("dir_path：",dir_path)
            #print(num+1,len(os.listdir(dir_path)))
            if num+1==len(os.listdir(dir_path)):
                print("第%s已下载"%(chapter_name))
                continue
            self.download_one_page(init_page,dir_path,0)
            chapter_href=self.download_chapter(browerdrive,dir_path,num)
            img_url_list.extend(chapter_href)
            chapter_info['href']=img_url_list
            chapter_info['chapter_name']=chapter_name
            cartoon_href_list.append(chapter_info)            
        return cartoon_href_list 

    def download_chapter(self,browerdrive,dir_path,max_num):
        '''
        下载一章节
        '''
        img_url=[]
        for x in range(0,max_num):
            browerdrive.find_element_by_class_name("img_land_next").click()
            wait=WebDriverWait(browerdrive,10)
            try:
                wait_element=wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR,"img[style='display: inline;']")))
                #href=browerdrive.find_element_by_css_selector("img[style='display: inline;']").get_attribute('src')
                href=wait_element.get_attribute('src')
                print("准备下载图片:",x+2)
                self.download_one_page(href,dir_path,x+1)
                img_url.append(href)
            except:
                print("wait失败!")
                continue
                        
        return img_url       

if  __name__=='__main__':
    down_load=Down_Cartoon() 
    down_load.run()