lxml_time_代理

import requests
from pyquery import PyQuery as pq
import json
import jsonpath
from lxml import etree
import os
import re
import time

html = '''
<div>
    <ul>
         <li class="item-0">first item</li>
         <li class="item-1"><a href="link2.html">second item</a></li>
         <li class="item-0 active"><a href="link3.html"><span class="bold">third item</span></a></li>
         <li class="item-1 active"><a href="link4.html">fourth item</a></li>
         <li class="item-0"><a href="link5.html">fifth item</a></li>
     </ul>
</div>
'''

# html = requests.get('http://news.4399.com/gonglue/lscs/kptj/').content.decode('gbk')
num = 0
# def pq方法(url):
#     global num
#     html=requests.get(url).content.decode('gbk')
#     doc = pq(html)
#     items = doc('#dq_list > li').items()
#     # print(doc)
#     # print(type(doc))
#     for item in items:
#         url=item.find('img').attr('lz_src')
#         num+=1
#         print(str(num),url)
#         url_content=requests.get(url).content
#         name = item.find('.kp-name').text()
        
#         with open('e:/py3/002/'+'{:0>4}'.format(str(num))+name+'.jpg','wb') as file:
#             file.write(url_content)
#         # print(url,name)

def transformCodec(re_data):#ascii (gbk) 转 unicode
    try:
        re_data = re_data.decode('gbk')
    except Exception as error:
        print (error)
        print ('delete illegal string,try again...')
        
        pos = re.findall(r'decodebytesinposition([d]+)-([d]+):illegal',str(error).replace(' ',''))
        if len(pos)==1:
            re_data = re_data[0:int(pos[0][0])]+re_data[int(pos[0][1]):]
            re_data = transformCodec(re_data)
            return re_data
    return re_data


def lxml方法(url):
    global num
    header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6756.400 QQBrowser/10.3.2473.400'}
    content=requests.get(url,headers=header).content
    html=content.decode('utf-8')
    # print(html.status_code)
    # print(content)
    # print(html)    
    r=etree.HTML(html)
    # items=r.xpath("//div[@class='box10-content']//ul[@id='dq_list']/li/a/img/@lz_src")
    items=r.xpath("//div[@id='list']/table//tr")
    # print(items)
    for item in items:
        dl_ip=item.xpath("./td[1]/text()")
        dl_port=item.xpath("./td[2]/text()")
        dl_name=item.xpath("./td[5]/text()")
        num+=1
        dl_ip=dl_ip[0]+":" if len(dl_ip)>=1 else ''
        dl_port=dl_port[0]+"#" if len(dl_port)>=1 else ''
        dl_name=dl_name[0] if len(dl_name)>=1 else ''
        
        # print(len(dl_ip))
        # print(dl_ip)
        # print(r'{}{}{}'.format(dl_ip,dl_port,dl_name))
        with open("proxy.txt",'a',encoding='utf-8') as file:
            file.write('{}{}{}
'.format(dl_ip,dl_port,dl_name))
        # lzcontent=requests.get(lzsrc).content
        # with open('e:/py3/004/'+'{:0>4}'.format(str(num))+'_'+kpname+'.jpg','wb')as file:
        #     file.write(lzcontent)





if __name__ == '__main__':
    with open("proxy.txt", 'w', encoding='utf-8') as file:
            file.write(str(time.localtime()[0])+'_'+str(time.localtime()[1])+'_'+str(time.localtime()[2])+'_采集：
')
    # url='https://www.kuaidaili.com/free/inha/1/'
    for i in range(1,11):
        print('第'+str(i)+'次：
')
        url2 = r'https://www.kuaidaili.com/free/inha/'+str(i)+r'/'
        print(url2)
        lxml方法(url2)
        time.sleep(5)

    # header={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6756.400 QQBrowser/10.3.2473.400'}
    # pq方法()
    # print(str(time.localtime()[0])+'_'+str(time.localtime()[1])+'_'+str(time.localtime()[2]))
    print(str(num)+' ok!')


    # 创建目录
    '''
    for dirnum in range(1,100):
        dirnum2='{:0>3}'.format(str(dirnum))
        mkpath="e:\py3\{}\".format(dirnum2)
        print(mkpath)
        print('已存在！') if os.path.exists(mkpath) else os.makedirs(mkpath)
    '''