Python爬虫快速上手教程

逃离我推掉我的手 2023-08-17 15:16 153阅读 0赞

# 1 这个是什么 #

整理Python中requests常用的API

# 2 代码 #

from bs4 import BeautifulSoup
    import requests
    import re
    
    # 使用requests进行网络求解，用BeautifulSoup处理html网页
    s_url = "https://www.baidu.com"
    o_header = {
            'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', 
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive',
            'Referer': 'https://www.baidu.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
    } 
    n_timeout = 36 # 设置超时秒数
    
    
    '''
    处理资源文件, url返回一个图片、音频、视频
    '''
    s_local_url = "img.jpg"
    resp_stream = requests.get(s_local_url, stream=True, headers=o_header, timeout=n_timeout) #进行图片下载   
    
    with open(s_local_url, 'wb', buffering = 1024) as fp:
        fp.write(resp_stream.content)
        fp.flush()          
    
    '''
    处理html, url返回一个网页
    '''
    # 使用get命令得到请求
    resp = requests.get(s_url, headers=o_header, timeout= n_timeout)
    
    
    # 得到明显的编码
    resp.encoding = resp.apparent_encoding
    
    
    # 字符串html转为soup
    soup__html = BeautifulSoup(resp, "lxml")   
    
    
    # 找到所有的id属性为abc的h
    soup__h = soup__html.find("a", id="h")
    print(soup__h.text)
    
    
    # 找到所有的class属性为abc的<img>
    soup__img_s = soup__html.find("img", class_="abc")
    for soup__img in soup__img_s:
        print(soup__img["src"], soup__img.text)
    
    
    # 找到所有的abc属性为opq的a
    soup__a = soup__html.find("a", attrs= {"abc" :"opq"})
    print(soup__a.text)
    
    
    # 找到所有的abc属性为opq 1, opq 2的a (正则找)
    soup__a = soup__html.find("a", attrs= {"abc" :re.compile(r"opq(\s\w+)?")})
    print(soup__a.text)

转载于:https://www.cnblogs.com/Kalafinaian/p/11440996.html