selenium3.0不用代理的情况下，获取异步请求的数据

太过爱你忘了你带给我的痛 2022-10-09 02:50 74阅读 0赞

最近爬取一个网站的时候，反爬比较厉害，各种弹窗，各种验证码，无限debugger，关键数据是ajax请求异步加载的。使用代理绕过前面几种反爬后，获取ajax的request和response成了头疼的问题，最终使用selenium的network日志分析来解决。为了方便以后使用，

**目录**

工具类：

使用方法：

控制台：

使用代理：

--------------------

### 工具类： ###

import json
    from selenium import webdriver
    from selenium.webdriver import DesiredCapabilities
    
    
    def get_xhr_logs(chrome):
        log_xhr_array = []
        for typelog in chrome.log_types:
            perfs = chrome.get_log(typelog)
            for row in perfs:
                log_data = row
                message_ = log_data['message']
                try:
                    log_json = json.loads(message_)
                    log = log_json['message']
                    if log['method'] == 'Network.responseReceived':
                        # 去掉静态js、css等，仅保留xhr请求
                        type_ = log['params']['type']
                        if type_ == "XHR":
                            log_xhr_array.append(log)
                except:
                    pass
        return log_xhr_array
    
    
    def get_log_options():
        option = webdriver.ChromeOptions()
        option.add_argument('--no-sandbox')
        option.add_argument('--headless')
        option.add_argument("--disable-extensions")
        option.add_argument("--allow-running-insecure-content")
        option.add_argument("--ignore-certificate-errors")
        option.add_argument("--disable-single-click-autofill")
        option.add_argument("--disable-autofill-keyboard-accessory-view[8]")
        option.add_argument("--disable-full-form-autofill-ios")
        option.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0')
        option.add_experimental_option('w3c', False)
        option.add_experimental_option('perfLoggingPrefs', {
            'enableNetwork': True,
            'enablePage': False,
        })
        return option
    
    
    def get_caps():
        caps = DesiredCapabilities.CHROME
        caps['loggingPrefs'] = {
            'browser': 'ALL',
            'performance': 'ALL',
        }
        caps['perfLoggingPrefs'] = {
            'enableNetwork': True,
            'enablePage': False,
            'enableTimeline': False
        }
        return caps

### 使用方法： ###

以访问[https://www.baidu.com/][https_www.baidu.com]为例：

import json
    import os
    import time
    
    # 导入工具类
    from myscrapy.network_log_option import *
    
    if __name__ == '__main__':
        # 使用工具类来获取options配置，而不是平时的webdriver.ChromeOptions()方法
        options = get_log_options()
        # 使用工具类来获取caps
        desired_capabilities = get_caps()
        # 这里也可以对options和caps加入其他的参数，比如代理参数等
        chrome = webdriver.Chrome(options=options, desired_capabilities=desired_capabilities)
        chrome.get("https://www.baidu.com/")
        chrome.maximize_window()
        # 用工具类来获取ajax请求日志
        logs = get_xhr_logs(chrome)
        for log in logs:
            print(log)
        chrome.quit()

### 控制台： ###

结果对应了百度的两条ajax请求：

C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\python.exe D:/pythonwork/myscrapy/myscrapy/myscrapy/test002.py
    {'method': 'Network.responseReceived', 'params': {'frameId': 'E6B2F71F5881A0C139C6405650B9B287', 'loaderId': 'AC357BD1718137E4CDBE450B8C470D1B', 'requestId': '39148.67', 'response': {'connectionId': 76, 'connectionReused': True, 'encodedDataLength': 117, 'fromDiskCache': False, 'fromPrefetchCache': False, 'fromServiceWorker': False, 'headers': {'Content-Length': '53', 'Content-Type': 'text/plain; charset=UTF-8', 'Date': 'Wed, 23 Jun 2021 02:35:04 GMT'}, 'headersText': 'HTTP/1.1 200 OK\r\nContent-Length: 53\r\nContent-Type: text/plain; charset=UTF-8\r\nDate: Wed, 23 Jun 2021 02:35:04 GMT\r\n\r\n', 'mimeType': 'text/plain', 'protocol': 'http/1.1', 'remoteIPAddress': '112.80.248.75', 'remotePort': 443, 'requestHeaders': {'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN', 'Connection': 'keep-alive', 'Cookie': 'BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q', 'Host': 'www.baidu.com', 'Referer': 'https://www.baidu.com/', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0', 'sec-ch-ua': '', 'sec-ch-ua-mobile': '?0'}, 'requestHeadersText': 'GET /sugrec?prod=pc_his&from=pc_web&json=1&sid=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350&hisdata=&_t=1624415704263&csor=0 HTTP/1.1\r\nHost: www.baidu.com\r\nConnection: keep-alive\r\nsec-ch-ua:\r\nAccept: application/json, text/javascript, */*; q=0.01\r\nsec-ch-ua-mobile: ?0\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0\r\nSec-Fetch-Site: same-origin\r\nSec-Fetch-Mode: cors\r\nSec-Fetch-Dest: empty\r\nReferer: https://www.baidu.com/\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-CN\r\nCookie: BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q\r\n', 'responseTime': 1624415704296.042, 'securityDetails': {'certificateId': 0, 'certificateTransparencyCompliance': 'unknown', 'cipher': 'AES_128_GCM', 'issuer': 'GlobalSign Organization Validation CA - SHA256 - G2', 'keyExchange': 'ECDHE_RSA', 'keyExchangeGroup': 'P-256', 'protocol': 'TLS 1.2', 'sanList': ['baidu.com', 'baifubao.com', 'www.baidu.cn', 'www.baidu.com.cn', 'mct.y.nuomi.com', 'apollo.auto', 'dwz.cn', '*.baidu.com', '*.baifubao.com', '*.baidustatic.com', '*.bdstatic.com', '*.bdimg.com', '*.hao123.com', '*.nuomi.com', '*.chuanke.com', '*.trustgo.com', '*.bce.baidu.com', '*.eyun.baidu.com', '*.map.baidu.com', '*.mbd.baidu.com', '*.fanyi.baidu.com', '*.baidubce.com', '*.mipcdn.com', '*.news.baidu.com', '*.baidupcs.com', '*.aipage.com', '*.aipage.cn', '*.bcehost.com', '*.safe.baidu.com', '*.im.baidu.com', '*.baiducontent.com', '*.dlnel.com', '*.dlnel.org', '*.dueros.baidu.com', '*.su.baidu.com', '*.91.com', '*.hao123.baidu.com', '*.apollo.auto', '*.xueshu.baidu.com', '*.bj.baidubce.com', '*.gz.baidubce.com', '*.smartapps.cn', '*.bdtjrcv.com', '*.hao222.com', '*.haokan.com', '*.pae.baidu.com', '*.vd.bdstatic.com', 'click.hm.baidu.com', 'log.hm.baidu.com', 'cm.pos.baidu.com', 'wn.pos.baidu.com', 'update.pan.baidu.com'], 'signedCertificateTimestampList': [], 'subjectName': 'baidu.com', 'validFrom': 1585811098, 'validTo': 1627277462}, 'securityState': 'secure', 'status': 200, 'statusText': 'OK', 'timing': {'connectEnd': -1, 'connectStart': -1, 'dnsEnd': -1, 'dnsStart': -1, 'proxyEnd': 0.455, 'proxyStart': 0.185, 'pushEnd': 0, 'pushStart': 0, 'receiveHeadersEnd': 31.722, 'requestTime': 325463.079675, 'sendEnd': 0.705, 'sendStart': 0.523, 'sslEnd': -1, 'sslStart': -1, 'workerFetchStart': -1, 'workerReady': -1, 'workerRespondWithSettled': -1, 'workerStart': -1}, 'url': 'https://www.baidu.com/sugrec?prod=pc_his&from=pc_web&json=1&sid=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350&hisdata=&_t=1624415704263&csor=0'}, 'timestamp': 325463.112169, 'type': 'XHR'}}
    {'method': 'Network.responseReceived', 'params': {'frameId': 'E6B2F71F5881A0C139C6405650B9B287', 'loaderId': 'AC357BD1718137E4CDBE450B8C470D1B', 'requestId': '39148.68', 'response': {'connectionId': 15, 'connectionReused': True, 'encodedDataLength': 360, 'fromDiskCache': False, 'fromPrefetchCache': False, 'fromServiceWorker': False, 'headers': {'Accept-Ranges': 'bytes', 'Cache-Control': 'max-age=315360000', 'Content-Encoding': 'gzip', 'Content-Length': '7112', 'Content-Type': 'application/javascript', 'Date': 'Wed, 23 Jun 2021 02:35:04 GMT', 'Etag': '"4451-4fdbd6734c340"', 'Expires': 'Sat, 21 Jun 2031 02:35:04 GMT', 'Last-Modified': 'Wed, 09 Jul 2014 07:10:29 GMT', 'Server': 'Apache', 'Vary': 'Accept-Encoding,User-Agent'}, 'headersText': 'HTTP/1.1 200 OK\r\nAccept-Ranges: bytes\r\nCache-Control: max-age=315360000\r\nContent-Encoding: gzip\r\nContent-Length: 7112\r\nContent-Type: application/javascript\r\nDate: Wed, 23 Jun 2021 02:35:04 GMT\r\nEtag: "4451-4fdbd6734c340"\r\nExpires: Sat, 21 Jun 2031 02:35:04 GMT\r\nLast-Modified: Wed, 09 Jul 2014 07:10:29 GMT\r\nServer: Apache\r\nVary: Accept-Encoding,User-Agent\r\n\r\n', 'mimeType': 'application/javascript', 'protocol': 'http/1.1', 'remoteIPAddress': '112.80.248.75', 'remotePort': 443, 'requestHeaders': {'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN', 'Connection': 'keep-alive', 'Cookie': 'BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q', 'Host': 'www.baidu.com', 'Referer': 'https://www.baidu.com/', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0', 'X-Requested-With': 'XMLHttpRequest', 'sec-ch-ua': '', 'sec-ch-ua-mobile': '?0'}, 'requestHeadersText': 'GET /cache/fpid/chromelib_1_1.js?_=1624415704024 HTTP/1.1\r\nHost: www.baidu.com\r\nConnection: keep-alive\r\nsec-ch-ua:\r\nAccept: text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01\r\nX-Requested-With: XMLHttpRequest\r\nsec-ch-ua-mobile: ?0\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0\r\nSec-Fetch-Site: same-origin\r\nSec-Fetch-Mode: cors\r\nSec-Fetch-Dest: empty\r\nReferer: https://www.baidu.com/\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-CN\r\nCookie: BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q\r\n', 'responseTime': 1624415704513.065, 'securityDetails': {'certificateId': 0, 'certificateTransparencyCompliance': 'unknown', 'cipher': 'AES_128_GCM', 'issuer': 'GlobalSign Organization Validation CA - SHA256 - G2', 'keyExchange': 'ECDHE_RSA', 'keyExchangeGroup': 'P-256', 'protocol': 'TLS 1.2', 'sanList': ['baidu.com', 'baifubao.com', 'www.baidu.cn', 'www.baidu.com.cn', 'mct.y.nuomi.com', 'apollo.auto', 'dwz.cn', '*.baidu.com', '*.baifubao.com', '*.baidustatic.com', '*.bdstatic.com', '*.bdimg.com', '*.hao123.com', '*.nuomi.com', '*.chuanke.com', '*.trustgo.com', '*.bce.baidu.com', '*.eyun.baidu.com', '*.map.baidu.com', '*.mbd.baidu.com', '*.fanyi.baidu.com', '*.baidubce.com', '*.mipcdn.com', '*.news.baidu.com', '*.baidupcs.com', '*.aipage.com', '*.aipage.cn', '*.bcehost.com', '*.safe.baidu.com', '*.im.baidu.com', '*.baiducontent.com', '*.dlnel.com', '*.dlnel.org', '*.dueros.baidu.com', '*.su.baidu.com', '*.91.com', '*.hao123.baidu.com', '*.apollo.auto', '*.xueshu.baidu.com', '*.bj.baidubce.com', '*.gz.baidubce.com', '*.smartapps.cn', '*.bdtjrcv.com', '*.hao222.com', '*.haokan.com', '*.pae.baidu.com', '*.vd.bdstatic.com', 'click.hm.baidu.com', 'log.hm.baidu.com', 'cm.pos.baidu.com', 'wn.pos.baidu.com', 'update.pan.baidu.com'], 'signedCertificateTimestampList': [], 'subjectName': 'baidu.com', 'validFrom': 1585811098, 'validTo': 1627277462}, 'securityState': 'secure', 'status': 200, 'statusText': 'OK', 'timing': {'connectEnd': -1, 'connectStart': -1, 'dnsEnd': -1, 'dnsStart': -1, 'proxyEnd': 1, 'proxyStart': 0.65, 'pushEnd': 0, 'pushStart': 0, 'receiveHeadersEnd': 53.016, 'requestTime': 325463.276445, 'sendEnd': 1.227, 'sendStart': 1.08, 'sslEnd': -1, 'sslStart': -1, 'workerFetchStart': -1, 'workerReady': -1, 'workerRespondWithSettled': -1, 'workerStart': -1}, 'url': 'https://www.baidu.com/cache/fpid/chromelib_1_1.js?_=1624415704024'}, 'timestamp': 325463.33218, 'type': 'XHR'}}

### ![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlbnhpbmdjaGVu_size_16_color_FFFFFF_t_70][]使用代理： ###

比如，使用chrom的远程调试代理：

--------------------

import json
    import os
    import time
    
    from myscrapy.network_log_option import *
    
    if __name__ == '__main__':
    
        # 启动代理命令,打开代理浏览器窗口
        cd_dir = r'cd C:\Users\Lenovo\AppData\Local\Google\Chrome\Application'
        start_proxy = r'chrome.exe --remote-debugging-port=9222 --user-data-dir="C:\selenum\AutomationProfile"'
        os.system(cd_dir)
        time.sleep(0.5)
        os.system(start_proxy)
    
        # 使用工具类来获取options配置，而不是平时的webdriver.ChromeOptions()方法
        options = get_log_options()
        # 配置代理参数
        options.add_experimental_option('debuggerAddress', 'localhost:90222')
        # 使用工具类来获取caps
        desired_capabilities = get_caps()
        # 这里也可以对options和caps加入其他的参数，比如代理参数等
        chrome = webdriver.Chrome(options=options, desired_capabilities=desired_capabilities)
        chrome.get("https://www.baidu.com/")
        chrome.maximize_window()
        # 用工具类来获取ajax请求日志
        logs = get_xhr_logs(chrome)
        for log in logs:
            print(log)
        chrome.quit()

[https_www.baidu.com]: https://www.baidu.com/
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlbnhpbmdjaGVu_size_16_color_FFFFFF_t_70]: /images/20221005/f27c2a69f48a4739a3113fd225efecaa.png