Python3.6爬虫+Djiago2.0+Mysql --数据爬取

1.下载对应版本的python mysql 模块 我的是：pymssql-2.2.0.dev0-cp36-cp36m-win_amd64.whl

2.手动创建table  

create table grilsbase
(
id int primary key auto_increment,
name varchar(50),height varchar(50),bwh varchar(50),title varchar(100),img_upload varchar(100),pc_img_upload varchar(100),
resource_id varchar(50),totals varchar(50),recommend_id varchar(50),date varchar(50),headimg_upload varchar(50),
show_datetime varchar(50),client_show_datetime varchar(50),video_duration varchar(50),free_select varchar(50),
trial_time varchar(50),viewtimes varchar(50),coop_customselect_654 varchar(50),coop_id varchar(50),tag_class varchar(50),
tag_name varchar(50),playerid varchar(50),block_detailid varchar(50),type varchar(50),istop varchar(50)
)

3.实现爬虫代码 

   导入模块：requests ，os，json，re,Mysqldb

   流程：获取数据=>分析数据=>解析数据=>持久化保存

#coding:utf-8
import  requests
import os
import json
import re
import MySQLdb
import threading
#获取数据url
gilsUrl='http://act.vip.xunlei.com/ugirls/js/ugirlsdata.js'
gilsDetailUrl='http://meitu.xunlei.com/detail.html'
gilsImgUrl='http://data.meitu.xunlei.com/data/image/%s/%s'
executor = threading.BoundedSemaphore(10)
regex=re.compile('/([^/]*?.jpg)$')
regexhead=re.compile('/([^/]*?).jpg$')
class MySQL:
    def __init__(self,host,user,pwd,db):
        self.host=host
        self.user=user
        self.db=db
        self.pwd=pwd
    def GetConnect(self):
        if not self.db:
            raise(NameError,'没有目标数据库')
        self.connect=MySQLdb.connect(host=self.host,user=self.user,password=self.pwd,database=self.db,port=3306,charset='utf8')
        cur=self.connect.cursor()
        if not cur:
            raise(NameError,'数据库访问失败')
        else:
            return cur
    def ExecSql(self,sql):
         cur=self.GetConnect()
         cur.execute(sql)
         self.connect.commit()
         self.connect.close()
    def ExecQuery(self,sql):
        cur=self.GetConnect()
        cur.execute(sql)
        resList = cur.fetchall()
        self.connect.close()
        return resList   

def getGirlsData():
    regex=re.compile("var ugirlsData=(.+)")
    r=requests.get(gilsUrl)
    jsond=regex.findall(r.text)
    with open('ugirlsdata.json','w+',encoding='utf-8') as f:
         f.write(jsond[0])
    #print('写入json成功')
    return json.loads(jsond[0])


def getImgName(imgurl):
    if(imgurl==''):
        return ''
    m=regex.findall(imgurl)
    if m is None:
        return ''
    else:
        return  m[0] if len(m)>0 else ''

def getImgNameHead(imgurl):
    if(imgurl==''):
        return ''
    m=regexhead.findall(imgurl)
    if m is None:
        return ''
    else:
        return  m[0] if len(m)>0 else ''

def WriteDB(jsdata):
    ms = MySQL(host="192.168.0.108", user="lin", pwd="12345678", db="grils")
    for data in jsdata:
        sql="insert into grilsbase(
        name,height,bwh,title,img_upload,pc_img_upload,resource_id,totals,recommend_id,
        date,headimg_upload,show_datetime,client_show_datetime,video_duration,free_select,trial_time,
        viewtimes,coop_customselect_654,coop_id,tag_class,tag_name,playerid,block_detailid,type,istop)
        values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % 
        (data['name'],data['height'],data['bwh'],data['title'],getImgName(data.get('img_upload','')),data['pc_img_upload'],data['resource_id'],data["totals"],data["recommend_id"], 
        data['date'],getImgName(data.get("headimg_upload",'')),data["show_datetime"],data["client_show_datetime"],data["video_duration"],data["free_select"],data["trial_time"], 
        data['viewtimes'],data['coop_customselect_654'],data['coop_id'],data.get('tag_class',''),data.get('tag_name',''),data.get('playerid',''),data['block_detailid'],data['type'],data['istop'])
        #print(sql)
        ms.ExecSql(sql)
        print('完成'+data['name']+'数据更新...')
        DownImg(data['name'],data["totals"],data['resource_id'],data["headimg_upload"],data["img_upload"])
        


def DownImg(name,totals,resource_id,headimg_upload,img_upload):
    path=creatFile(resource_id)
    if headimg_upload.strip()!='':
        #os.remove('./pic/'+resource_id+'/'+getImgName(headimg_upload)+'.jpg')
        DownImgRun(headimg_upload,path,getImgNameHead(headimg_upload))
    if img_upload.strip()!='':
        #os.remove('./pic/'+resource_id+'/'+getImgName(img_upload)+'.jpg')
        DownImgRun(img_upload,path,getImgNameHead(img_upload))
    #print('正在下载'+name+'图片')
    
    for i in range(1,int(totals)+1):
        url=gilsImgUrl%(resource_id,str(i)+'.jpg')
        DownImgRun(url,path,i)
        #t=threading.Thread(target=DownImgRun,args={url,path,i})
        #t.start()
        #t.join()
    
        

def DownImgRun(url,path,i):
    #print(url)
    
    r=requests.get(url)
    if(r.status_code==200):
        with open(path+'/'+str(i)+'.jpg','wb') as fimg:
            fimg.write(r.content)
            


def creatFile(dirname):
    path='./pic/'+dirname
    if os.path.exists(path):
        return path
    else:
        os.makedirs(path)
        return  path


if __name__ == '__main__':
    gri=getGirlsData()
    WriteDB(gri)

4.运行效果 和结果