环境:
fiddler 搭建本地代理,代理地址为 127.0.0.1:8888
python 版本:3.6.7
requests 版本:2.21.0
1. 执行的测试python脚本为:
requests.get('http://localhost:5000/test',proxies={'http':'127.0.0.1:8888'},headers={'Cookie':'c=d'},cookies={'a':'b'})
以上脚本同时在headers和get方法的参数中设置了cookie
2.
get方法位于 requests/api.py
内容为:
def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary, list of tuples or bytes to send in the body of the :class:`Request`. :param **kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object :rtype: requests.Response """ kwargs.setdefault('allow_redirects', True) return request('get', url, params=params, **kwargs)
可看到返回request方法的值,
一系列参数传给了Session对象的request方法:
1 def request(method, url, **kwargs): 2 """Constructs and sends a :class:`Request <Request>`. 3 :param method: method for the new :class:`Request` object. 4 :param url: URL for the new :class:`Request` object. 5 :param params: (optional) Dictionary, list of tuples or bytes to send 6 in the body of the :class:`Request`. 7 :param data: (optional) Dictionary, list of tuples, bytes, or file-like 8 object to send in the body of the :class:`Request`. 9 :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. 10 :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. 11 :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. 12 :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. 13 ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` 14 or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string 15 defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers 16 to add for the file. 17 :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. 18 :param timeout: (optional) How many seconds to wait for the server to send data 19 before giving up, as a float, or a :ref:`(connect timeout, read 20 timeout) <timeouts>` tuple. 21 :type timeout: float or tuple 22 :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``. 23 :type allow_redirects: bool 24 :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. 25 :param verify: (optional) Either a boolean, in which case it controls whether we verify 26 the server's TLS certificate, or a string, in which case it must be a path 27 to a CA bundle to use. Defaults to ``True``. 28 :param stream: (optional) if ``False``, the response content will be immediately downloaded. 29 :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. 30 :return: :class:`Response <Response>` object 31 :rtype: requests.Response 32 Usage:: 33 >>> import requests 34 >>> req = requests.request('GET', 'https://httpbin.org/get') 35 <Response [200]> 36 """ 37 38 # By using the 'with' statement we are sure the session is closed, thus we 39 # avoid leaving sockets open which can trigger a ResourceWarning in some 40 # cases, and look like a memory leak in others. 41 with sessions.Session() as session: 42 return session.request(method=method, url=url, **kwargs)
实例化一个Request对象然后prepare_request:
1 def request(self, method, url, 2 params=None, data=None, headers=None, cookies=None, files=None, 3 auth=None, timeout=None, allow_redirects=True, proxies=None, 4 hooks=None, stream=None, verify=None, cert=None, json=None): 5 """Constructs a :class:`Request <Request>`, prepares it and sends it. 6 Returns :class:`Response <Response>` object. 7 :param method: method for the new :class:`Request` object. 8 :param url: URL for the new :class:`Request` object. 9 :param params: (optional) Dictionary or bytes to be sent in the query 10 string for the :class:`Request`. 11 :param data: (optional) Dictionary, list of tuples, bytes, or file-like 12 object to send in the body of the :class:`Request`. 13 :param json: (optional) json to send in the body of the 14 :class:`Request`. 15 :param headers: (optional) Dictionary of HTTP Headers to send with the 16 :class:`Request`. 17 :param cookies: (optional) Dict or CookieJar object to send with the 18 :class:`Request`. 19 :param files: (optional) Dictionary of ``'filename': file-like-objects`` 20 for multipart encoding upload. 21 :param auth: (optional) Auth tuple or callable to enable 22 Basic/Digest/Custom HTTP Auth. 23 :param timeout: (optional) How long to wait for the server to send 24 data before giving up, as a float, or a :ref:`(connect timeout, 25 read timeout) <timeouts>` tuple. 26 :type timeout: float or tuple 27 :param allow_redirects: (optional) Set to True by default. 28 :type allow_redirects: bool 29 :param proxies: (optional) Dictionary mapping protocol or protocol and 30 hostname to the URL of the proxy. 31 :param stream: (optional) whether to immediately download the response 32 content. Defaults to ``False``. 33 :param verify: (optional) Either a boolean, in which case it controls whether we verify 34 the server's TLS certificate, or a string, in which case it must be a path 35 to a CA bundle to use. Defaults to ``True``. 36 :param cert: (optional) if String, path to ssl client cert file (.pem). 37 If Tuple, ('cert', 'key') pair. 38 :rtype: requests.Response 39 """ 40 # Create the Request. 41 req = Request( 42 method=method.upper(), 43 url=url, 44 headers=headers, 45 files=files, 46 data=data or {}, 47 json=json, 48 params=params or {}, 49 auth=auth, 50 cookies=cookies, 51 hooks=hooks, 52 ) 53 prep = self.prepare_request(req) 54 55 proxies = proxies or {} 56 57 settings = self.merge_environment_settings( 58 prep.url, proxies, stream, verify, cert 59 ) 60 61 # Send the request. 62 send_kwargs = { 63 'timeout': timeout, 64 'allow_redirects': allow_redirects, 65 } 66 send_kwargs.update(settings) 67 resp = self.send(prep, **send_kwargs) 68 69 return resp
实例化一个PreparedRequest对象然后prepare(此时cookie还没受影响):
1 def prepare_request(self, request): 2 """Constructs a :class:`PreparedRequest <PreparedRequest>` for 3 transmission and returns it. The :class:`PreparedRequest` has settings 4 merged from the :class:`Request <Request>` instance and those of the 5 :class:`Session`. 6 :param request: :class:`Request` instance to prepare with this 7 session's settings. 8 :rtype: requests.PreparedRequest 9 """ 10 cookies = request.cookies or {} 11 12 # Bootstrap CookieJar. 13 if not isinstance(cookies, cookielib.CookieJar): 14 cookies = cookiejar_from_dict(cookies) 15 16 # Merge with session cookies 17 merged_cookies = merge_cookies( 18 merge_cookies(RequestsCookieJar(), self.cookies), cookies) 19 20 # Set environment's basic authentication if not explicitly set. 21 auth = request.auth 22 if self.trust_env and not auth and not self.auth: 23 auth = get_netrc_auth(request.url) 24 25 p = PreparedRequest() 26 p.prepare( 27 method=request.method.upper(), 28 url=request.url, 29 files=request.files, 30 data=request.data, 31 json=request.json, 32 headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), 33 params=merge_setting(request.params, self.params), 34 auth=merge_setting(auth, self.auth), 35 cookies=merged_cookies, 36 hooks=merge_hooks(request.hooks, self.hooks), 37 ) 38 return p
prepare时的操作:
1 def prepare(self, 2 method=None, url=None, headers=None, files=None, data=None, 3 params=None, auth=None, cookies=None, hooks=None, json=None): 4 """Prepares the entire request with the given parameters.""" 5 6 self.prepare_method(method) 7 self.prepare_url(url, params) 8 self.prepare_headers(headers) 9 self.prepare_cookies(cookies) 10 self.prepare_body(data, files, json) 11 self.prepare_auth(auth, url) 12 13 # Note that prepare_auth must be last to enable authentication schemes 14 # such as OAuth to work on a fully prepared request. 15 16 # This MUST go after prepare_auth. Authenticators could add a hook 17 self.prepare_hooks(hooks)
prepare_headers,此时self.headers['Cookie']会被赋值:
1 def prepare_headers(self, headers): 2 """Prepares the given HTTP headers.""" 3 4 self.headers = CaseInsensitiveDict() 5 if headers: 6 for header in headers.items(): 7 # Raise exception on invalid header value. 8 check_header_validity(header) 9 name, value = header 10 self.headers[to_native_string(name)] = value
prepare_cookies,此步骤中关键在于cookie_header 变量是不是None:
1 def prepare_cookies(self, cookies): 2 """Prepares the given HTTP cookie data. 3 This function eventually generates a ``Cookie`` header from the 4 given cookies using cookielib. Due to cookielib's design, the header 5 will not be regenerated if it already exists, meaning this function 6 can only be called once for the life of the 7 :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls 8 to ``prepare_cookies`` will have no actual effect, unless the "Cookie" 9 header is removed beforehand. 10 """ 11 if isinstance(cookies, cookielib.CookieJar): 12 self._cookies = cookies 13 else: 14 self._cookies = cookiejar_from_dict(cookies) 15 16 cookie_header = get_cookie_header(self._cookies, self) 17 if cookie_header is not None: 18 self.headers['Cookie'] = cookie_header
可看到get_cookie_header 函数中先模拟一个请求对象,然后将cookie添加近该模拟的请求对象中,然后获取其header的Cookie:
1 def get_cookie_header(jar, request): 2 """ 3 Produce an appropriate Cookie header string to be sent with `request`, or None. 4 :rtype: str 5 """ 6 r = MockRequest(request) 7 jar.add_cookie_header(r) 8 return r.get_new_headers().get('Cookie')
以上代码中jar是一个cookielib.CookieJar类的对象,根据requests/compat.py 文件中中的内容可得知在python3中cookielib是 from http import cookiejar as cookielib 得来的,在python的http/cookiejar.py 源码中,可看到
add_cookie_header方法只有在request的headers中没有Cookie的时候把cookie的信息添加进headers:
1 def add_cookie_header(self, request): 2 """Add correct Cookie: header to request (urllib.request.Request object). 3 The Cookie2 header is also added unless policy.hide_cookie2 is true. 4 """ 5 _debug("add_cookie_header") 6 self._cookies_lock.acquire() 7 try: 8 9 self._policy._now = self._now = int(time.time()) 10 11 cookies = self._cookies_for_request(request) 12 13 attrs = self._cookie_attrs(cookies) 14 if attrs: 15 if not request.has_header("Cookie"): 16 request.add_unredirected_header( 17 "Cookie", "; ".join(attrs)) 18 19 # if necessary, advertise that we know RFC 2965 20 if (self._policy.rfc2965 and not self._policy.hide_cookie2 and 21 not request.has_header("Cookie2")): 22 for cookie in cookies: 23 if cookie.version != 1: 24 request.add_unredirected_header("Cookie2", '$Version="1"') 25 break 26 27 finally: 28 self._cookies_lock.release() 29 30 self.clear_expired_cookies()
如上文所述,同时在headers中和方法参数中设置cookie的时候只有headers中的cookie有效。