xpath的使用

墨蓝 2023-09-30 19:05 43阅读 0赞

### 基本使用 ###

# _*_ coding : utf-8 _*_
    # @Time : 2023/1/28 14:48
    # @Author : 李阶熊
    # @File : 解析xpath的基本使用
    # @Project : pythonProject
    from lxml import etree
    
    # xpath 解析
    # （1） 本地文件                                                              etree.parse
    # （2） 服务器响应的数据  response.read().decode('utf-8')                       etree.HTML()
    
    # xpath解析本地文件
    tree = etree.parse('../html/xpath_test.html')
    print(tree)
    
    # tree.xpath('xpath路径')
    
    # 查找ul下面的li
    li_list = tree.xpath('//ul/li')
    
    print(li_list)
    print(len(li_list))
    
    
    # 查找所有有id的属性的li标签
    li_list = tree.xpath('//ul/li[@id]/text()')
    print(li_list)
    print(len(li_list))
    
    
    # 找到id为l1的li标签
    li_list = tree.xpath('//ul/li[@id="l1"]/text()')
    print(li_list)
    print(len(li_list))
    
    # 查找id为l1的li标签的class的属性值
    li = tree.xpath('//ul/li[@id="l1"]/@class')
    print(li)
    print(len(li))
    
    # 查询id中包含l的li标签
    li_list = tree.xpath('//ul/li[contains(@id,"l")]/text()')
    print(li_list)
    print(len(li_list))
    
    # 查询id的值以l开头的li标签
    li_list = tree.xpath('//ul/li[starts-with(@id,"l")]/text()')
    print(li_list)
    print(len(li_list))
    
    # 查询id为l1喝class为a3的
    li_list = tree.xpath('//ul/li[@id="a3" and @class="a3"]/text()')
    print(li_list)
    print(len(li_list))
    
    li_list = tree.xpath('//ul/li[@id="a3"]/text()' | '//ul/li[@id="l1"]/text()')
    print(li_list)
    print(len(li_list))

### 使用xpath获取百度一下 ###

# _*_ coding : utf-8 _*_
    # @Time : 2023/1/28 15:50
    # @Author : 李阶熊
    # @File : 获取百度网站的百度一下
    # @Project : pythonProject
    # 1、获取网页原阿门
    # 2、解析  解析的服务器响应的文件  etree.HTML
    # 3、打印
    
    import urllib.request
    
    url = "https://www.baidu.com/"
    
    headers = {
        
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
    }
    
    request = urllib.request.Request(url=url, headers=headers)
    
    # handler = urllib.request.HTTPHandler
    #
    # opener = urllib.request.build_opener(handler)
    #
    # response = opener.open(request)
    
    response = urllib.request.urlopen(request)
    
    # 获取网页源码
    content = response.read().decode('utf-8')
    
    # 解析网页源码  来获取我们想要的数据
    from lxml import etree
    
    # 解析服务器响应的文件
    tree = etree.HTML(content)
    
    # 获取想要的数据  xpath的返回值是一个列表类型的数据
    result = tree.xpath('//input[@id="su"]/@value')
    
    print(result[0])

### 站长素材图片获取和下载 ###

# _*_ coding : utf-8 _*_
    # @Time : 2023/1/28 16:12
    # @Author : 李阶熊
    # @File : 站长素材
    # @Project : pythonProject
    # （1） 请求对象的定制
    # （2） 获取网页的源码
    # （3） 下载
    # https://sc.chinaz.com/tu/fengjing.html
    # https://sc.chinaz.com/tu/fengjing-3-0-0.html
    import urllib.request
    from lxml import etree
    import urllib.error
    
    last = '.html'
    url = 'https://sc.chinaz.com/tu/fengjing'
    
    headers = {
        
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
    }
    
    
    def Schedule(a, b, c):
        per = 100.0 * a * b / c
        if per > 100:
            per = 100
            print('完成！')
        print('%.2f%%' % per)
    
    
    def create_request(p):
        if p == 1:
            current_url = url + last
        else:
            current_url = url + '-' + str(p) + '-0-0' + last
            print(current_url)
        request1 = urllib.request.Request(url=current_url, headers=headers)
        return request1
    
    
    def get_content(request2):
        response = urllib.request.urlopen(request2)
        content1 = response.read().decode('utf-8')
        return content1
    
    
    def down_load(content1):
        # 下载图片
        # urllib.request.urlretrieve('图片地址', '文件名称')
        tree = etree.HTML(content1)
        name_list = tree.xpath('//div[@id="ulcontent"]//a/img/@alt')
    
        # 一般设计到图片的网站，会有懒加载
        src_list = tree.xpath('//div[@id="ulcontent"]//a/img/@data-src')
        for i in range(len(name_list)):
            try:
    
                name = name_list[i] + '.jpg'
                src = src_list[i]
                url_1 = 'http:' + src
                url_1 = url_1.replace('\\', '/')
                print(url_1)
                urllib.request.urlretrieve(url_1, './images/' + name, Schedule)
            except urllib.error.HTTPError:
                print('图片路径有错误哦')
                continue
    
    
    if __name__ == '__main__':
        start_page = int(input('请输入起始页码'))
        end_page = int(input('请输入结束页码'))
        for page in range(start_page, end_page + 1):
            # (1) 请求对象的定制
            request = create_request(page)
            # (2)获取网页的源码
            content = get_content(request)
            # （3） 下载
            down_load(content)