通过python爬取笔趣阁小说，获取图片保存本地，数据保存mysql

àì夳堔傛蜴生んèń 2023-05-28 06:55 4阅读 0赞

**通过python爬取小说图片、小说名称、作者、章节、内容。直接上代码**

import requests
    import time
    from bs4 import BeautifulSoup
    from queue import Queue
    import threading
    import pymysql
    from DBUtils.PooledDB import PooledDB
    
    proxy='124.243.226.18:8888'
    
    #如果代理需要验证，只需要在前面加上用户名密码，如下所示
    
    # proxy='username:password@124.243.226.18:8888'
    proxies={ 
        'http':'http://119.140.186.17:894',
        'http':'http://27.24.20.22:766',
        'http':'http://114.98.189.158:766',
        'http':'http://114.101.246.185:23564'
    }
    
    pool = PooledDB(creator=pymysql,
                            maxconnections=0,  # 连接池允许的最大连接数，0和None表示不限制连接数
                            mincached=10,  # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
                            maxcached=0,  # 链接池中最多闲置的链接，0和None不限制
    
                            blocking=True,  # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
                            host='47.106.122.177',  # 数据库ip地址
                            port=3306,
                            user='root',
                            passwd='123456',
                            db='zskxt',
                            use_unicode=True,
                            charset='utf8')
    
    
    send_headers = { 
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
        "Connection": "keep-alive",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.8"
    }  # 伪装成浏览器
    
    #获取小说URL列表
    def huoquxiaoshuo():
        search_real_url = 'http://www.xbiquge.la/xiaoshuodaquan/'
        try:
            html = requests.get(search_real_url).text
    
            htmlTree = BeautifulSoup(html, 'html.parser')
            url_list=htmlTree.find(id='main').find_all('a')
            print(url_list[0]);
            urlreutrn = []
            for url in url_list:
                urlreutrn.append(url.attrs['href'])
            return urlreutrn
        except Exception as e:
            print(e)
    
    class Producer(threading.Thread):
        def __init__(self, url_queue , *args, **kwargs):
            super(Producer, self).__init__(*args, **kwargs)
            self.url_queue = url_queue
    
        def run(self):
    
            # 获取小说目录
            try:
                while True:
                    if self.url_queue.empty():
                        break
    
                    url=self.url_queue.get()
                    while True:
                        try:
                            text = requests.get(url,send_headers)
                        except Exception as e:
                            print(e)
                            continue
                        text.encoding = 'utf-8'
                        novel_source=text.text
                        htmlTree = BeautifulSoup(novel_source, 'html.parser')
                        print(htmlTree.title);
    
                        if str(htmlTree.title).find('503')==-1:
                            break
                        else :
                            time.sleep(1)
    
                    img = htmlTree.find(id='fmimg').find_all('img')
                    #print(img)
                    #保存图片
                    while True:
                        try:
                            response = requests.get(img[0].get('src'),send_headers)
                        except Exception as e:
                            print(e)
                            continue
                        # 获取的文本实际上是图片的二进制文本
                        imgxr = response.content
                        if len(imgxr) <300:
                            time.sleep(1)
                        else :
                            break
    
                    # 将他拷贝到本地文件 w 写 b 二进制 wb代表写入二进制文本
                    imgxrlist=[]
                    imgxrlist=img[0].get('src').split("/")
                    bookmsimg=imgxrlist[len(imgxrlist) - 1]
                    with open('d:\\img\\'+bookmsimg, 'wb') as f:
                        f.write(imgxr)
                    id=url.split("/")
                    bookid=id[len(id) - 2]
                    bookname=htmlTree.find(id='info').find('h1').string
                    bookzz = htmlTree.find(id='info').find_all('p')[0].string.replace('作 者：','')
                    bookms = htmlTree.find(id='intro').find_all('p')[1].string
                    bookms=bookms.replace("'","\'")
                    sql="insert into zsk_book_sm (bookid,bookname,zz,zhgxsj,msnr,tp) values ('xbqg"+bookid+"','"+bookname+"','"+bookzz+"',now(),'"+bookms+"','"+bookmsimg+"')"
                    db = pool.connection()  # 连接数据池
                    cursor = db.cursor()  # 获取游标
                    try:
                        cursor.execute(sql)
                        db.commit()
                    except Exception as e:
                        print("SQL ERROR！", e)
                        db.rollback()
                    #finally:
                        #cursor.close()
                        #db.close()
    
                    bookzjlisturl=[]
    
                    bookzjlist=htmlTree.find(id='list').find_all('a')
                    for url in bookzjlist:
                        urlzj=str(url.get('href'))
                        zjid=urlzj.split("/")[3].replace(".html", "")
                        #获取章节内容
                        while True:
                            try:
                                time.sleep(1)
                                #nrtext = requests.get('http://www.xbiquge.la/2/2210/1744317.html', send_headers)
    
                                nrtext = requests.get('http://www.xbiquge.la'+urlzj, send_headers)
                                #nrtext = requests.get('http://www.xbiquge.la' + urlzj, send_headers, proxies=proxies)
                            except Exception as e:
                                print(e)
                                continue
                            nrtext.encoding = 'utf-8'
                            nrnovel_source = nrtext.text
                            nrhtmlTree = BeautifulSoup(nrnovel_source, 'html.parser')
                            print(nrhtmlTree.title);
                            if str(nrhtmlTree.title).find('503') == -1:
                                break
                            else:
                                time.sleep(1)
                        content=nrhtmlTree.find(id='content')
                        #if str(content).index("</p>")!=-1:
                        # nrhtmlTree.find('p').decompose()
                        a=str(content).replace("<div id=\"content\">","").replace("</div>","")
                        a=a.replace("'","")
                        nrsql = "INSERT INTO `zsk_book_content` (`id`, `content`) VALUES ('" +zjid + "', '" + a + "')"
                        zjsql ="INSERT INTO zsk_book_ml (mlid, zjmc, bookid, content_id, is_delete, words) VALUES ('"+ zjid + "', '" + url.text + "', '" +bookid+ "', '" + urlzj.split("/")[3].replace(".html", "") + "', 0, 1)";
                        try:
                            cursor.execute(zjsql)
                            cursor.execute(nrsql)
                            db.commit()
                        except Exception as e:
                            print("SQL ERROR！", e)
                            db.rollback()
                            #finally:
                            #cursor.close()
                            #db.close()
    
            except Exception as e:
                print(e)
    
    if __name__ == '__main__':
        url_list=huoquxiaoshuo();
        print(url_list[0]);
        url_queue = Queue(len(url_list))
    
        for url in url_list:
            url_queue.put(url)
    
        for x in range(3):
            t = Producer(url_queue)
            t.start()
    
        # book_name = input('请输入你要下载的小说名字(确保输入的小说名字正确)：')
        # novel_url, novel_name = download(book_name)
        # chapter_list = get_chapter(novel_url)
        # get_content(chapter_list, novel_name)

**数据库结构如下**

Target Server Type    : MYSQL
    Target Server Version : 50562
    File Encoding         : 65001
    
    Date: 2020-03-28 12:32:08
    */
    
    SET FOREIGN_KEY_CHECKS=0;
    -- ----------------------------
    -- Table structure for `zsk_book_content`
    -- ----------------------------
    DROP TABLE IF EXISTS `zsk_book_content`;
    CREATE TABLE `zsk_book_content` (
    `id`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '' COMMENT '内容id' ,
    `content`  text CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '章节内容' ,
    PRIMARY KEY (`id`)
    )
    ENGINE=InnoDB
    DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
    
    ;
    
    -- ----------------------------
    -- Table structure for `zsk_book_ml`
    -- ----------------------------
    DROP TABLE IF EXISTS `zsk_book_ml`;
    CREATE TABLE `zsk_book_ml` (
    `mlid`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT '目录id' ,
    `zjmc`  varchar(200) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '目录章节名称' ,
    `bookid`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL DEFAULT '' COMMENT '书id' ,
    `content_id`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '文章内容id' ,
    `is_delete`  int(11) NULL DEFAULT NULL COMMENT '删除标识' ,
    `words`  int(11) NULL DEFAULT NULL COMMENT '字数' ,
    `ctime`  timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '创建时间' ,
    `mtime`  timestamp NOT NULL DEFAULT '0000-00-00 00:00:00' COMMENT '修改时间' ,
    PRIMARY KEY (`mlid`, `bookid`)
    )
    ENGINE=InnoDB
    DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
    
    ;
    
    -- ----------------------------
    -- Table structure for `zsk_book_sm`
    -- ----------------------------
    DROP TABLE IF EXISTS `zsk_book_sm`;
    CREATE TABLE `zsk_book_sm` (
    `bookid`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT '书id' ,
    `bookname`  varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '书名' ,
    `zz`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '作者' ,
    `zhgxsj`  timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '最后更新时间' ,
    `tp`  varchar(500) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '封面' ,
    `zxzj`  varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '最新章节' ,
    `zxzjid`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '最新章节id' ,
    `msnr`  varchar(500) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '描述内容' ,
    `lb`  varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '类别' ,
    PRIMARY KEY (`bookid`)
    )
    ENGINE=InnoDB
    DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
    
    ;