Python爬取多个点击事件的网页数据

今天药忘吃喽~ 2023-01-17 07:45 166阅读 0赞

### 文章目录 ###

*   *  1、环境准备
     *  2、源代码

## 1、环境准备 ##

pip install openpyxl
    pip install requests
    pip install re

## 2、源代码 ##

import openpyxl
    import requests
    import re
    
    from bs4 import BeautifulSoup
    
    # 创建Excel表并写入数据
    wb = openpyxl.Workbook()  # 创建Excel对象
    ws = wb.active  # 获取当前正在操作的表对象
    # 往表中写入标题行,以列表形式写入！
    ws.append(['事件名称', '时间', '地点名称', '事件简介','事件详情'])
    
    # 要爬取的网页
    baseurl = 'http://www.xxx/'
    
    # 请求头
    headers = { 
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0',
        'Upgrade-Insecure-Requests': '1'
    }
    
    ip = '58.218.200.220:2551'
    
    proxies = { 
        'http': 'http://' + ip,
        'https': 'https://' + ip,
    }
    
    EventName = []  # 事件名称
    Introduction = [] # 事件简介
    Detail = [] # 事件详情
    Time = [] # 时间
    all_detail = '' # 详情
    
    # 定义发起请求函数
    def urlBS(url,proxies):
        resp = requests.get(url=url,proxies=proxies)
        html = resp.content.decode('utf-8') # 设置编码格式
        soup = BeautifulSoup(html, 'lxml') # 解析网页
        return soup
    
    
    soup = urlBS(baseurl,proxies)  #调用函数
    lis = soup.find('div', class_="dyw605_text_list").find_all('li') # 从网页获取的信息
    # print(lis)
    
    for i in range(3,len(lis)):
        newurl = lis[i].find('a')['href']
        # print(newurl)
    
        # 请求每篇文章
        result = urlBS(newurl,proxies)  #调用函数
        last_url = result.find('div', class_="yd_title_text").find('a')['href'] # 获取网址
        # print(last_url)
        title = result.find('div', class_="yd_title_text").find('h4').get_text() # 获取标题
        # print(title)
        new_title = re.findall('(.+)（',title)
        EventName.append(new_title[0])
    
        # time = re.compile('（(.+)）').findall(title)
        time = re.findall('（(.+)）',title)
        # print(time)
        Time.append(time[0])
    
        introduction = result.find('div', class_="yd_title_text").find('span').get_text()   # 获取简介
        # print(introduction)
        Introduction.append(introduction)
    
        result_url = urlBS(last_url,proxies)  # 调用函数
        detail = result_url.find('div', class_="font_area_mid").find_all('p')[1:]  # 获取详情 除去第一个p标签后的数据
        # print(detail)
        for info in detail:
            all_detail = info.get_text() + all_detail
        # print(all_detail)
        # new_detail = re.findall('\xa0\xa0\xa0\xa0(.+)\xa0\xa0\xa0\xa0',all_detail)
        new_detail = re.sub('\xa0\xa0\xa0\xa0',' ',all_detail)
        Detail.append(new_detail)
    
    
    print("EventName",EventName)
    print("Introduction",Introduction)
    print("Detail",Detail)
    print("Time",Time)
    
    for i in range(len(EventName)):  # 写入工作表中
        ws.append([EventName[i], Time[i], str("北京"), Introduction[i],Detail[i]])
    
    wb.save('会议数据2.xlsx')  # 存入所有信息后，保存为filename.xlsx
    print("保存成功！")