Selenium中如何抓取网络请求响应及WebSocket信息

我们在使用Selenium测试Web或Electronjs/Cef框架应用时，有时候操作一个元素需要判断是否发送了请求以及请求的参数是否正确

我们可以通过，开启Chrome的性能日志来然后配合driver.get_log("performance")来查看请求，然后对Network相关的日子进行过滤，
实现如下：

获取Chrome性能日志

import json
from pprint import pprint
from selenium import webdriver

caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},   # 记录性能日志
    'goog:chromeOptions': {'extensions': [], 'args': ['--headless']}  # 无界面模式
}

driver = webdriver.Chrome(desired_capabilities=caps)

driver.get('https://httpbin.org/get')
logs = driver.get_log("performance")
for item in logs:
    log = json.loads(item["message"])["message"]
    pprint(log)
    if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        pprint(log)

运行结果如下:

{'method': 'Network.responseReceived',
 'params': {'frameId': '2445B94E9E1DB51A1B1F4F3B0A3F03F5',
            'loaderId': 'D0DE1754D5C5F1E54DC3B0DB2A09ADD6',
            'requestId': 'D0DE1754D5C5F1E54DC3B0DB2A09ADD6',
            'response': {'connectionId': 0,
                         'connectionReused': False,
                         'encodedDataLength': -1,
                         'fromDiskCache': False,
                         'fromPrefetchCache': False,
                         'fromServiceWorker': False,
                         'headers': {'Content-Type': 'text/plain;charset=US-ASCII'},
                         'mimeType': 'text/plain',
                         'protocol': 'data',
                         'remoteIPAddress': '',
                         'remotePort': 0,
                         'securityState': 'secure',
                         'status': 200,
                         'statusText': 'OK',
                         'url': 'data:,'},
            'timestamp': 57524.763168,
            'type': 'Document'}}
{'method': 'Network.requestWillBeSent',
 'params': {'documentURL': 'https://httpbin.org/get',
            'frameId': '2445B94E9E1DB51A1B1F4F3B0A3F03F5',
            'hasUserGesture': False,
            'initiator': {'type': 'other'},
            'loaderId': '8BB61F3D2448E8BC91A4A5AD7E690673',
            'request': {'headers': {'Upgrade-Insecure-Requests': '1',
                                    'User-Agent': 'Mozilla/5.0 (Macintosh; '
                                                  'Intel Mac OS X 10_15_7) '
                                                  'AppleWebKit/537.36 (KHTML, '
                                                  'like Gecko) '
                                                  'HeadlessChrome/91.0.4472.114 '
                                                  'Safari/537.36'},
                        'initialPriority': 'VeryHigh',
                        'method': 'GET',
                        'mixedContentType': 'none',
                        'referrerPolicy': 'strict-origin-when-cross-origin',
                        'url': 'https://httpbin.org/get'},
            'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673',
            'timestamp': 57524.961438,
            'type': 'Document',
            'wallTime': 1626501610.512192}}
{'method': 'Network.requestWillBeSentExtraInfo',
 'params': {'associatedCookies': [],
            'headers': {':authority': 'httpbin.org',
                        ':method': 'GET',
                        ':path': '/get',
                        ':scheme': 'https',
                        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                        'accept-encoding': 'gzip, deflate, br',
                        'accept-language': 'en-US',
                        'sec-fetch-dest': 'document',
                        'sec-fetch-mode': 'navigate',
                        'sec-fetch-site': 'none',
                        'sec-fetch-user': '?1',
                        'upgrade-insecure-requests': '1',
                        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X '
                                      '10_15_7) AppleWebKit/537.36 (KHTML, '
                                      'like Gecko) '
                                      'HeadlessChrome/91.0.4472.114 '
                                      'Safari/537.36'},
            'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673'}}
{'method': 'Network.responseReceivedExtraInfo',
 'params': {'blockedCookies': [],
            'headers': {'access-control-allow-credentials': 'true',
                        'access-control-allow-origin': '*',
                        'content-length': '754',
                        'content-type': 'application/json',
                        'date': 'Sat, 17 Jul 2021 06:00:11 GMT',
                        'server': 'gunicorn/19.9.0'},
            'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673',
            'resourceIPAddressSpace': 'Public'}}
{'method': 'Network.responseReceived',
 'params': {'frameId': '2445B94E9E1DB51A1B1F4F3B0A3F03F5',
            'loaderId': '8BB61F3D2448E8BC91A4A5AD7E690673',
            'requestId': '8BB61F3D2448E8BC91A4A5AD7E690673',
            'response': {'connectionId': 12,
                         'connectionReused': False,
                         'encodedDataLength': 123,
                         'fromDiskCache': False,
                         'fromPrefetchCache': False,
                         'fromServiceWorker': False,
                         'headers': {'access-control-allow-credentials': 'true',
                                     'access-control-allow-origin': '*',
                                     'content-length': '754',
                                     'content-type': 'application/json',
                                     'date': 'Sat, 17 Jul 2021 06:00:11 GMT',
                                     'server': 'gunicorn/19.9.0'},
                         'mimeType': 'application/json',
                         'protocol': 'h2',
                         'remoteIPAddress': '52.201.75.114',
                         'remotePort': 443,
                         'requestHeaders': {':authority': 'httpbin.org',
                                            ':method': 'GET',
                                            ':path': '/get',
                                            ':scheme': 'https',
                                            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                                            'accept-encoding': 'gzip, deflate, '
                                                               'br',
                                            'accept-language': 'en-US',
                                            'sec-fetch-dest': 'document',
                                            'sec-fetch-mode': 'navigate',
                                            'sec-fetch-site': 'none',
                                            'sec-fetch-user': '?1',
                                            'upgrade-insecure-requests': '1',
                                            'user-agent': 'Mozilla/5.0 '
                                                          '(Macintosh; Intel '
                                                          'Mac OS X 10_15_7) '
                                                          'AppleWebKit/537.36 '
                                                          '(KHTML, like Gecko) '
                                                          'HeadlessChrome/91.0.4472.114 '
                                                          'Safari/537.36'},
                         'responseTime': 1626501611316.694,
                         'securityDetails': {'certificateId': 0,
                                             'certificateTransparencyCompliance': 'unknown',
                                             'cipher': 'AES_128_GCM',
                                             'issuer': 'Amazon',
                                             'keyExchange': 'ECDHE_RSA',
                                             'keyExchangeGroup': 'P-256',
                                             'protocol': 'TLS 1.2',
                                             'sanList': ['httpbin.org',
                                                         '*.httpbin.org'],
                                             'signedCertificateTimestampList': [],
                                             'subjectName': 'httpbin.org',
                                             'validFrom': 1608508800,
                                             'validTo': 1642636799},
                         'securityState': 'secure',
                         'status': 200,
                         'statusText': '',
                         'timing': {'connectEnd': 548.386,
                                    'connectStart': 26.524,
                                    'dnsEnd': 26.524,
                                    'dnsStart': 14.11,
                                    'proxyEnd': -1,
                                    'proxyStart': -1,
                                    'pushEnd': 0,
                                    'pushStart': 0,
                                    'receiveHeadersEnd': 803.146,
                                    'requestTime': 57524.962922,
                                    'sendEnd': 548.745,
                                    'sendStart': 548.611,
                                    'sslEnd': 548.36,
                                    'sslStart': 277.934,
                                    'workerFetchStart': -1,
                                    'workerReady': -1,
                                    'workerRespondWithSettled': -1,
                                    'workerStart': -1},
                         'url': 'https://httpbin.org/get'},
            'timestamp': 57525.76746,
            'type': 'Document'}}

获取请求及响应信息

由于日志中没有接口后台数据和响应数据，我们可以通过执行cdp名利获取，修改后代码如下

import json
from pprint import pprint
from selenium import webdriver
from selenium.common.exceptions import WebDriverException


caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},
    'goog:chromeOptions': {'extensions': [], 'args': ['--headless']}
}

driver = webdriver.Chrome(desired_capabilities=caps)

driver.get('https://httpbin.org/get')
logs = driver.get_log("performance")
for item in logs:
    log = json.loads(item["message"])["message"]
    # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
    if log["method"] == 'Network.responseReceived':
        url = log['params']['response']['url']
        if url == 'data:,':  # 过滤掉初始data页面，后续可以根据 log['params']['response']['type']过滤请求类型
            continue
        print('请求', url)
        request_id = log['params']['requestId']

        request_headers = log['params']['response']['requestHeaders']
        response_headers = log['params']['response']['headers']
        response_time = log['params']['response']['responseTime']
        status_code = log['params']['response']['status']

        try:
            request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
        except WebDriverException:  # 没有后台数据获取时会有异常
            request_data = None

        response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
        print('响应', response_body)

执行后显示如下：

请求 https://httpbin.org/get
响应 {
  "args": {}, 
  "headers": {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 
    "Accept-Encoding": "gzip, deflate, br", 
    "Accept-Language": "en-US", 
    "Cache-Control": "max-age=0", 
    "Host": "httpbin.org", 
    "Sec-Fetch-Dest": "document", 
    "Sec-Fetch-Mode": "navigate", 
    "Sec-Fetch-Site": "none", 
    "Sec-Fetch-User": "?1", 
    "Upgrade-Insecure-Requests": "1", 
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/91.0.4472.114 Safari/537.36", 
    "X-Amzn-Trace-Id": "Root=1-60f2dd9d-6533f9526707f25f7d6c38de"
  }, 
  "origin": "123.118.150.190", 
  "url": "https://httpbin.org/get"
}

参考：How to Capture Network Traffic When Scraping with Selenium & Python
Chrome DevTools Protocol