基于python第三方requests 模块的HTTP请求类

使用requests模块构造的下载器,首先安装第三方库requests

pip install requests

 1 class StrongDownload(object):
    
    def __init__(self):
        #拿到代理iplist
        self.iplist = ['自己想办法搞'] #
        self.UserAgent = ['自己想办法搞']  
    
    def get(self,url,timeout,proxy=False,num_retries=3):
        '''url timeout,proxy,num_retries
                返回response对象
        '''
        #伪造UserAgent
        UA = self.UserAgent[random.randint(0,len(self.UserAgent)-1)]
        #UA = random.choice(self.UserAgent)
        headers = {'User-Agent':UA}
        if proxy == False:
            try:
                return requests.get(url,timeout=timeout,headers=headers)
            except Exception as e:
                print e
                #判断retries
                if num_retries>0:
                    print u'爬取网页错误,10s后继续尝试.....'
                    time.sleep(1)
                    return self.get(url, timeout, False, num_retries-1)
                else:#重试次数用尽,使用代理
                    time.sleep(1)
                    IP = str(random.choice(self.iplist))
                    proxy = {'http',IP}
                    return self.get(url, timeout, True,5)
                
        else:#使用代理
            
            try:
                print '开始使用代理'
                IP = random.choice(self.iplist)
                proxy = {'http':IP}
                print proxy['http']
                return requests.get(url,headers=headers, timeout=timeout, proxies=proxy)
            except:
                if num_retries>0:
                    print u'代理爬取网页错误,10s后继续尝试.....'
                    time.sleep(1)
                    return self.get(url, timeout,True, num_retries-1)
                else:#代理也不行
                    time.sleep(30)
                    print u'代理也没用...'
                    return None