PYTHON流向下载

#-*- coding:utf-8 -*-
import gzip
import re
import http.cookiejar
import urllib.request
import urllib.parse
import xlwt
import time,os
 


def saveexcel(flow,filename,coding='gbk'):
    #flow 需要转换为excel的里面，格式为双层列表
    #coding excel页面编码
    try:
        workbook = xlwt.Workbook(encoding=coding)
        sheet = workbook.add_sheet('Sheet1')
        for row,rowdata in enumerate(flow):
            for col,val in enumerate(rowdata):
                sheet.write(row,col,val.strip(),style = xlwt.Style.default_style)
        excelname = '\%s.xls'%filename
        workbook.save(excelname)
        return excelname

    except Exception as e:
        if hasattr(e,"code"):
            print ('excel写入失败，错误原因' +str(e.code))
        if hasattr(e,"reason"):
            print ('excel写入失败，错误原因' +str(e.reason))
        return None

 #从指定页面中取表单参数
def getParm(data,parm):
    cer = re.compile('name="'+parm+'".* value="(.*?)"', flags = 0)
    strlist = cer.findall(data)
    
    if strlist:
        return strlist[0]
    else:
        return None

def getOpener():
    #自动设置COOKIER
    # deal with the Cookies
    print( '正在设置cookie')    
    cj = http.cookiejar.CookieJar()
    pro = urllib.request.HTTPCookieProcessor(cj)
    opener = urllib.request.build_opener(pro, urllib.request.HTTPHandler)
    urllib.request.install_opener(opener)    
    print( '设置cookie成功')        
    return opener
 

header = {
    'Connection': 'Keep-Alive',
    'Accept': 'text/html, application/xhtml+xml, */*',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko'
}
 
url = 'http://115.231.58.130:8021/Default.aspx'
header['Referer']='http://115.231.58.130:8021/'
#1、设置Cookie
opener = getOpener()


##2、初始化数据开始
request = urllib.request.Request(url)
try:
    html = urllib.request.urlopen(request).read()
    #取表单参数
    EVENTVALIDATION = getParm(html.decode('gbk'),'__EVENTVALIDATION')
    VIEWSTATEGENERATOR =  getParm(html.decode('gbk'),'__VIEWSTATEGENERATOR')
    VIEWSTATE = getParm(html.decode('gbk'),'__VIEWSTATE')
    btnsubmit = getParm(html.decode('gbk'),'sbtnSubmit')   

except urllib.request.URLError as e:
    if hasattr(e,"code"):
        print ('请求页面失败，请检查网络设置，错误原因' +str(e.code))
    if hasattr(e,"reason"):
        print ('请求页面失败，请检查网络设置，错误原因' +str(e.reason))
#取表单参数结束 


id = '***'
password = '***'
postDict = {
         'LoginID':id,
        'Pwd':password,
        '__EVENTVALIDATION':EVENTVALIDATION,
        '__VIEWSTATEGENERATOR':VIEWSTATEGENERATOR,
        '__VIEWSTATE':VIEWSTATE,
        'btnSubmit':btnsubmit
}
postData = urllib.parse.urlencode(postDict).encode(encoding='UTF8')

##3、正式登录
request = urllib.request.Request(url, postData,headers=header)
try:
    response = urllib.request.urlopen(request)
    data = response.read()
except urllib.request.URLError as e:
    if hasattr(e,"code"):
        print ('页面加载失败，请检查网络及账号设置，错误原因' +str(e.code))
    if hasattr(e,"reason"):
        print ('页面加载失败，请检查网络及账号设置，错误原因' +str(e.reason))

#登录结束
print('login:',data.decode('gbk'))

##4 进入产品搜索界面进行数据提取

posturl= 'http://115.231.58.130:8021/Search/ProductFlow.aspx'
request = urllib.request.Request(posturl)
try:
    html = urllib.request.urlopen(request).read()
    #取表单参数
    EVENTVALIDATION = getParm(html.decode('gbk'),'__EVENTVALIDATION')
    VIEWSTATEGENERATOR =  getParm(html.decode('gbk'),'__VIEWSTATEGENERATOR')
    VIEWSTATE = getParm(html.decode('gbk'),'__VIEWSTATE')
    #btnsearch = getParm(html.decode('gbk'),'btnSearcht')
except urllib.request.URLError as e:
    if hasattr(e,"code"):
        print ('请求页面失败，请检查网络设置，错误原因' +str(e.code))
    if hasattr(e,"reason"):
        print ('请求页面失败，请检查网络设置，错误原因' +str(e.reason))


postDict['__EVENTTARGET']=''
postDict['__EVENTARGUMENT']=''  
postDict['__EVENTVALIDATION']=EVENTVALIDATION
postDict['__VIEWSTATEGENERATOR']=VIEWSTATEGENERATOR
postDict['__VIEWSTATE']=VIEWSTATE
postDict['PName']=''
postDict['PID']=''
postDict['txtStartDate']='2016-01-01'
postDict['txtEndDate']='2016-01-31'
postDict['ConvertToExcel.x']='6'
postDict['ConvertToExcel.y']='9'
postDict['btnSearch']='' 
postData = urllib.parse.urlencode(postDict).encode(encoding='UTF8')

print( '搜索页面数据获取成功，正在抓取流向数据...')    


###登录搜索页面
request = urllib.request.Request(posturl, postData,headers=header)
try:
    response = urllib.request.urlopen(request)
    data = response.readlines()
except urllib.request.URLError as e:
    if hasattr(e,"code"):
        print ('页面加载失败，请检查网络及账号设置，错误原因' +str(e.code))
    if hasattr(e,"reason"):
        print ('页面加载失败，请检查网络及账号设置，错误原因' +str(e.reason))
print( '流向抓取成功，正在保存为excel...')
print('search:',data)
##5 保存为excel

workbook = xlwt.Workbook(encoding='gbk')
sheet = workbook.add_sheet('Sheet1')
for row,rowdata in enumerate(data):
    rowdata_list = rowdata.decode('gbk').split('	')
    for col,val in enumerate(rowdata_list):
            sheet.write(row,col,val,style = xlwt.Style.default_style)

ntime = time.strftime('%Y%m%d%H%M%S')
excelname = ntime+'%s.xls'%'宁波宝瑞达'
workbook.save(excelname)
print( 'excel导出成功，请查看程序目录下%s文件。'%excelname)