python爬取公司Wiki(confluence)静态信息

绝地灬酷狼 2022-08-29 10:04 525阅读 0赞

# -*- coding:utf-8 -*-
    from  urllib import request,parse
    import re
    
    class BaiduTiebaSpider:
        "定义常用的变量"
        def __init__(self):
            self.url = "http://wiki.enflame.cn/pages/viewpage.action?spaceKey=SQA&title=DevOps"
            self.headers = {
        
                'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36',
                'Cookie': 'mywork.tab.tasks=false; JSESSIONID=C2B4CFF43A0D62E97E64D4FEB989AB9C'
            }
    
        def get_html(self):
            "获取相应内容的函数"
            req = request.Request(url=self.url,headers=self.headers)
            res = request.urlopen(req)
            html = res.read().decode('utf-8')
            return html
    
        def parse_html(self,html):
            "解析提取数据的函数"
            pattern = re.compile('<li>.*?>(.*?)</a></li>')
            r_list = pattern.findall(html)
            print(r_list)
    
    
    
        def run(self):
            "程序入口函数"
            html = self.get_html()
            self.parse_html(html)
    
    
    if __name__ == '__main__':
        spider = BaiduTiebaSpider()
        spider.run()
    
    输出结果：
    C:\Users\shiya.liu\PycharmProjects\pythonProject\venv\Scripts\python.exe C:/Users/shiya.liu/PycharmProjects/pythonProject/Reptile/wiki交差版.py
    ['转至内容', '转至导航栏', '转至主菜单', '转至动作菜单', '转至快速搜索', '00. 决策', '01. 实验室', '02. 服务', '03. 分享', '99. Achieve', 'Software Quality Committee</a>\xa0</li><li>OKR：<a href="/display/HR/_DevOps">_DevOps']
    
    Process finished with exit code 0
    
    暂时做个记录 有待完善