Python多进程爬去猫眼TOP100电影数据存入Mongodb数据库

忘是亡心i 2024-04-19 15:00 67阅读 0赞

## 需要准备的原材料： ##

1.  Mongodb数据库，去官网下载安装包，最新版本的数据库支持一键式安装不用配置启动服务了
2.  pyquery库 `pip install pyquery`
3.  pymongo库 `pip install pymongo`
4.  requests库 `pip install requests`

## 代码分析 ##

1、导入需要的库

import requests
    from pyquery import PyQuery as py
    import multiprocessing 
    from pymongo import MongoClient
    import time

2、连接数据库

def get_mongo(host="localhost", port=27017, dbname="mydb", collection="colletion9"):
    #     1.连接数据库
        client = MongoClient(host, port)
    #     2.创建数据库
        db = client[dbname]
    #     3.创建collection
        collection = db[collection]
        return collection

3、编写爬虫程序

def maoyan_spider(page):
        headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36",
        }
        url = "https://maoyan.com/board/4?offset=" + str(page*10)  # 分析猫眼的url规律
        # 1.拿到网站的数据
        response = requests.get(url, headers=headers).text
        
        # 3.转换成pyquery对象
        doc = py(response)
        
        # 4.拿到一页影名、评分等数据类标签内的数据
        movies = doc(".board-item-content").items()
        colletion = get_mongo(dbname="Maoyan")  # 链接数据库
        
        # 5.拿到每个电影的数据
        for movie in movies:
            name = movie(".name a").text()
            actor = movie(".star").text()
            pub_time = movie(".releasetime").text()
            score = movie(".integer").text() + movie(".fraction").text()
            movie_data = {"name": name, "actor": actor, "pub_time": pub_time, "score": score}
    
    		# 将数据插入数据库
            colletion.insert_one(movie_data)  
    
    		# 这条语句放在插入前和插入后打印的结果不一样，这在里会把 存入数据库的id打印出来
            print(movie_data)

4、开启多进程爬取数据

def main():
        time1 = time.time()
        for page in range(0, 5):
            p1 = multiprocessing.Process(target=maoyan_spider, args=(2*page,))  #爬去偶数页
            p2 = multiprocessing.Process(target=maoyan_spider, args=(2*page+1,))  # 爬取奇数页
            p1.start()
            p2.start()
        time2 = timpe.time()
        print(time2-time1)
    
    
    if __name__ == "__main__":
        main()

5、也可以采用进程池的方式开启多进程

def main(): 
        # 创建一个可以容纳5个进程的进程池
        ps=Pool(5)
        for page in range(10):
            
            # ps.apply(worker,args=(i,))          # 同步执行
            ps.apply_async(maoyan_spider,args=(page,))  # 异步执行
    
        # 关闭进程池，停止接受其它进程
        ps.close()
        # 阻塞进程
        ps.join()
        
    
    if __name__ == '__main__':
        main()