Python四线程爬取西刺代理

import requests
from bs4 import BeautifulSoup 
import lxml
import telnetlib #验证代理的可用性
import pymysql.cursors
import random
import threading




BASEURL = 'http://www.xicidaili.com/' #西刺首页
urls = [BASEURL+ 'nn/',BASEURL+'nt/',BASEURL+'wn/',BASEURL+'wt/']#西刺分组(more)的ip信息链接列表

#请求头信息，必须有User-Agent
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}

#proxies = {'https': 'http://123.57.85.224:80', 'http': 'http://123.57.85.224:80'}

#获得与数据库的连接和游标
def get_cc():
    # 连接MySQL数据库
    connection = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='root', db='iptables', 
                                 charset='utf8', cursorclass=pymysql.cursors.DictCursor)
    # 通过cursor创建游标
    cursor = connection.cursor()
    return connection,cursor



#保存ip_port到数据库
def save_ip_port(ip_port):
    connection,cursor = get_cc()
    try:
        sql = 'insert into iptable(ip_port) values("'+ip_port+'")'
        cursor.execute(sql)
    except:
        print('保存'+ip_port+'失败!!!!!')
    else:
        connection.commit()
        connection.close()



#从数据库获得ip_port
def get_ip_port():
    connection,cursor = get_cc()
    sql_get_id = 'select id,ip_port from iptable'
    cursor.execute(sql_get_id)
    #fetchone()是查询一条数据
    id_list = cursor.fetchall()#得到所有的id的字典列表
    i = random.randint(0,len(id_list)-1)
    id_num = id_list[i]['id']
    ip_port = id_list[i]['ip_port'] #获得所有可用的代理
    
    return id_num,ip_port#返回id和ip_port：192.168.1.2:8080

#删除被封的ip_port
def del_ip_port(id_num):
    connection,cursor = get_cc()
    try:
        sql = 'delete from iptable where id = ' + str(id_num)
        cursor.execute(sql)
    except:
        print('删除'+ip_port+'失败!!!!!')
    else:
        connection.commit()
        connection.close()



#获得代理
def get_proxies(ip_port):#ip_port = '192.168.2.45:8088'
    proxy_ip = 'http://' + ip_port
    proxy_ips = 'https://' + ip_port
    proxies = {'https': proxy_ips, 'http': proxy_ip}
    return proxies


#获得对应url分类的最大页码
def get_max_pagenum(url): #url是more(分类)的链接，/nn,/nt....
    
    response = requests.get(url,headers = headers)
    status_code = response.status_code
    soup = BeautifulSoup(response.content,'lxml')
    max_pagenum = soup.find('div',attrs = {'class':'pagination'}).find_all('a')[-2].string
    max_pagenum = int(max_pagenum)
    return max_pagenum

#验证代理是否有用,ip_port = '192.168.2.45:8088'
#每得到一个ip_port都要进行验证，如果可用则保存，否则抛弃
def verifyProxyList(ip_port):
    url = 'http://www.baidu.com'
    # proxies = { "http": "http://"+ ip_port }
    host ,port = ip_port.split(':')
    try:
        # res = requests.get(url,headers = headers,proxies = proxies,timeout = 5.0)
        telnetlib.Telnet(host, port=port, timeout=5)
    except:
        print('---Failur:' + ip_port)
    else:
        #ips.append(ip_port)#这里应该存储到Redis等数据库中
        save_ip_port(ip_port)
        
        

def main(url,proxies):#这里是more的链接,/nn/1,/nn/2....
    
    try:
        response = requests.get(url,headers = headers,proxies = proxies,timeout = 5.0)
        status_code = response.status_code #503说明ip被封

        if(status_code != requests.codes.ok):#响应的不是正常状态
            #删除旧的代理ip_port,这里还需要验证是否有bug
            old_ip_port = proxies['http'][7:]
            del_ip_port(old_ip_port)
            #修改代理，重新请求
            id_num,ip_port = get_ip_port()
            proxies = get_proxies(ip_port)
            print(str(proxies))
            return
                      
        soup = BeautifulSoup(response.content,'lxml')
            
        results = soup.find_all('tr')#遍历所有的tr
         
        for result in results[1:]:#这里第一个tr子标签是th，所以会报错
            tdlist = result.find_all('td')
            ip_port = tdlist[1].string+':'+tdlist[2].string
            verifyProxyList(ip_port)
    except:
       print('请求异常......')

class myThread(threading.Thread):
    def __init__(self, threadID, name, url):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.url = url
    
    
    
    def run(self):
        print('正在执行线程：'+self.name)#没有验证这一行的可行性
        id_num,ip_port = get_ip_port()
        proxies = get_proxies(ip_port)
        max_pagenum = get_max_pagenum(self.url)
        #print(max_pagenum)
        for i in range(1,max_pagenum):
            url = self.url + '/' + str(i)
            main(url,proxies)
        
#4线程爬取西刺的ip代理池
if __name__ == '__main__':
   
    t1 = myThread(1,"Thread-1",urls[0])
    t2 = myThread(2,"Thread-2",urls[1])
    t3 = myThread(3,"Thread-3",urls[2])
    t4 = myThread(4,"Thread-4",urls[3])
    t1.start()
    t2.start()
    t3.start()
    t4.start()
    t1.join()
    t2.join()
    t3.join()
    t4.join()