爬取imdb资料库

傷城~ 2023-05-31 06:55 5阅读 0赞

写一个简单的python爬虫程序，爬取imdb资料库，将爬取到的电影信息，存储到一个excel表格中

因为imdb资料库电影网没有反扒措施，直接爬取即可

代码如下：

import requests
    from lxml import etree
    import pandas as pd
    import numpy as np
    
    # 第一页：'http://www.imdb.cn/IMDB250/'
    # 第一页：'http://www.imdb.cn/imdb250/2'
    # 第三页：'http://www.imdb.cn/imdb250/3'
    
    class IMDB:
    
        def __init__(self,move_name_list,move_score_list,move_bieming_list,move_ywm_list,move_dir_list,move_time_list):
            self.move_name_list = move_name_list
            self.move_score_list = move_score_list
            self.move_bieming_list = move_bieming_list
            self.move_ywm_list = move_ywm_list
            self.move_dir_list = move_dir_list
            self.move_time_list = move_time_list
            self.spider()
    
        def shuzu(self,content):
            '''
            将列表转化为二维数组
            :param content: 列表
            :return:  二维数组
            '''
            content = np.array(content)
            content = content.reshape(len(content),1)
            return content
    
        def pankong(self,content):
            """
            判空函数
            :param content: 列表
            :return: 字符串
            """
            if len(content) > 0:
                return content[0]
            else:
                return '无'
    
        def spider(self):
            '''
            爬虫程序
            :return:
            '''
            header = {
                'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
                'Host': 'www.imdb.cn'
            }
            for page in range(1,10):
                if page == 1:
                    url = 'http://www.imdb.cn/IMDB250/'
                else:
                    url = f'http://www.imdb.cn/imdb250/{page}'
                response = requests.get(url=url,headers=header).content.decode('utf-8')
                tree = etree.HTML(response)
                a_list = tree.xpath('//div[@class="ss-3 clear"]/a')
                print(len(a_list))
                for a in a_list:
                    # 电影名称
                    move_name = a.xpath('.//div[@class="honghe-3"]/p/text()')
                    move_name = self.pankong(move_name)
                    self.move_name_list.append(move_name)
                    print(move_name)
                    # 电影评分
                    move_score = a.xpath('.//div[@class="honghe-2"]/span/i/text()')
                    move_score =  self.pankong(move_score)
                    self.move_score_list.append(move_score)
                    print(move_score)
                    # 电影别名
                    move_bieming = a.xpath('.//div[@class="honghe-4 clear"]/p[1]/i/text()')
                    move_bieming =  self.pankong(move_bieming)
                    self.move_bieming_list.append(move_bieming)
                    print(move_bieming)
                    # 电影英文名
                    move_ywm = a.xpath('.//div[@class="honghe-4 clear"]/p[2]/text()')
                    move_ywm = self.pankong(move_ywm)
                    self.move_ywm_list.append(move_ywm)
                    print(move_ywm)
                    # 导演
                    move_dir = a.xpath('.//div[@class="honghe-4 clear"]/p[3]/span/text()')
                    move_dir = self.pankong(move_dir)
                    self.move_dir_list.append(move_dir)
                    print(move_dir)
                    # 年份
                    move_time = a.xpath('.//div[@class="honghe-4 clear"]/p[4]/i[3]/text()')
                    move_time = self.pankong(move_time)
                    self.move_time_list.append(move_time)
                    print(move_time)
            # 将所有列表转换为二维数组
            self.move_name_list = self.shuzu(self.move_name_list)
            self.move_score_list = self.shuzu(self.move_score_list)
            self.move_bieming_list = self.shuzu(self.move_bieming_list)
            self.move_ywm_list = self.shuzu(self.move_ywm_list)
            self.move_dir_list = self.shuzu(self.move_dir_list)
            self.move_time_list = self.shuzu(self.move_time_list)
            # 将二维数组进行数据类型转换，并存到excel表格中
            self.save_data(self.move_name_list,self.move_score_list,self.move_bieming_list,self.move_ywm_list,self.move_dir_list,self.move_time_list)
    
        def save_data(self,s1,s2,s3,s4,s5,s6):
            '''
            建立dataFrame数据类型,并将所有数据存到excel表格中
            :return:
            '''
            # 将所有二维数组进行横向拼接
            content = np.concatenate((s1,s2,s3,s4,s5,s6),axis=1)
            print(content)
            print(content.shape)
            # 将拼接后的二维数组转化为dataFframe数据类型
            data = pd.DataFrame(content,columns=['电影名称','评分','别名','英文名','导演','上映时间'],index=range(1,248))
            # 在同级目录下创建IMDBtop250电影信息1.xlsx文件，并将数据存到该文件中
            data.to_excel('IMDBtop250电影信息1.xlsx',index=False)
    
    if __name__ == '__main__':
        move_name_list = []
        move_score_list = []
        move_bieming_list = []
        move_ywm_list = []
        move_dir_list = []
        move_time_list = []
        IMDB(move_name_list,move_score_list,move_bieming_list,move_ywm_list,move_dir_list,move_time_list)

运行结果：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDIzOTU0MQ_size_16_color_FFFFFF_t_70][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDIzOTU0MQ_size_16_color_FFFFFF_t_70]: /images/20230531/ca7d1b970c5c4f19835bc5bfb66acdb7.png