基于百度飞桨的皮影戏生成器

朱雀 2022-10-13 12:54 203阅读 0赞

## 前言 ##

飞桨(PaddlePaddle)是集深度学习核心框架、工具组件和服务平台为一体的技术先进、功能完备的开源深度学习平台，已被中国企业广泛使 用，深度契合企业应用需求，拥有活跃的开发者社区生态。提供丰富的官方支持模型集合，我们这里将要使用到其中的骨骼节点检测模型， 通过PaddleHub提供的人体骨骼关键点检测预训练模型，我们就可以快速实现皮影戏的效果。  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjAyMDc4OA_size_16_color_FFFFFF_t_70]  
PaddleHub可以便捷地获取PaddlePaddle生态下的预训练模型，完成模型的管理和一键预测。配合使用Fine-tune API，可以基于大规模预训练模型快速完成迁移学习，让预训练模型能更好地服务于用户特定场景的应用。

## 安装飞桨（Python环境） ##

进入cmd界面后  
**安装PaddlePaddle**

python -m pip install paddlepaddle==2.0.2 -i https://mirror.baidu.com/pypi/simple

**安装PaddleHub**

pip install PaddleHub

**安装人体骨骼关键节点检测模型：通过PaddleHub来安装人体骨骼关键点检测模型：human\_pose\_estimation\_resnet50\_mpii**

hub install human_pose_estimation_resnet50_mpii==1.1.1

## 素材准备 ##

<table> 
 <thead> 
  <tr> 
   <th>文件夹</th> 
   <th>用途</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>background</td> 
   <td>存放背景图</td> 
  </tr> 
  <tr> 
   <td>example</td> 
   <td>界面图片</td> 
  </tr> 
  <tr> 
   <td>work\mp4_img</td> 
   <td>视频素材切割帧图</td> 
  </tr> 
  <tr> 
   <td>work\mp4_img_analysis</td> 
   <td>合并帧图</td> 
  </tr> 
  <tr> 
   <td>work\shadow_play_material</td> 
   <td>皮影人关键部位图片</td> 
  </tr> 
  <tr> 
   <td>work\test1.MP4</td> 
   <td>视频文件</td> 
  </tr> 
  <tr> 
   <td>成品</td> 
   <td>生成视频</td> 
  </tr> 
 </tbody> 
</table>

皮影戏人关键部位素材图片请到这个网址中下载，：https://aistudio.baidu.com/aistudio/projectdetail/764130?fromQRCode=1&shared=1

## 效果视频 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjAyMDc4OA_size_16_color_FFFFFF_t_70 1]

https://www.bilibili.com/video/BV1Hq4y177B6/

## 完整代码 ##

from tkinter import *
    from tkinter import ttk  # 导入ttk模块，因为下拉菜单控件在ttk中
    import tkinter
    import os
    import cv2
    import time
    from tkinter import filedialog
    from handright import Template, handwrite
    from PIL import Image, ImageFont, ImageTk
    import tkinter.messagebox  # 要使用messagebox先要导入模块
    import paddlehub as hub
    import matplotlib.pyplot as plt
    from matplotlib.image import imread
    import numpy as np
    
    def show_img(img_path, size=8):
        ''' 文件读取图片显示 '''
        im = imread(img_path)
        plt.figure(figsize=(size, size))
        plt.axis("off")
        plt.imshow(im)
    
    
    def img_show_bgr(image, size=8):
        ''' cv读取的图片显示 '''
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        plt.figure(figsize=(size, size))
        plt.imshow(image)
    
        plt.axis("off")
        plt.show()
    
    pose_estimation = hub.Module(name="human_pose_estimation_resnet50_mpii")
    
    def get_true_angel(value):
        ''' 转转得到角度值 '''
        return value / np.pi * 180
    
    
    def get_angle(x1, y1, x2, y2):
        ''' 计算旋转角度 '''
        dx = abs(x1 - x2)
        dy = abs(y1 - y2)
        result_angele = 0
        if x1 == x2:
            if y1 > y2:
                result_angele = 180
        else:
            if y1 != y2:
                the_angle = int(get_true_angel(np.arctan(dx / dy)))
            if x1 < x2:
                if y1 > y2:
                    result_angele = -(180 - the_angle)
                elif y1 < y2:
                    result_angele = -the_angle
                elif y1 == y2:
                    result_angele = -90
            elif x1 > x2:
                if y1 > y2:
                    result_angele = 180 - the_angle
                elif y1 < y2:
                    result_angele = the_angle
                elif y1 == y2:
                    result_angele = 90
    
        if result_angele < 0:
            result_angele = 360 + result_angele
        return result_angele
    
    
    def rotate_bound(image, angle, key_point_y):
        ''' 旋转图像，并取得关节点偏移量 '''
        # 获取图像的尺寸
        (h, w) = image.shape[:2]
        # 旋转中心
        (cx, cy) = (w / 2, h / 2)
        # 关键点必须在中心的y轴上
        (kx, ky) = cx, key_point_y
        d = abs(ky - cy)
    
        # 设置旋转矩阵
        M = cv2.getRotationMatrix2D((cx, cy), -angle, 1.0)
        cos = np.abs(M[0, 0])
        sin = np.abs(M[0, 1])
    
        # 计算图像旋转后的新边界
        nW = int((h * sin) + (w * cos))
        nH = int((h * cos) + (w * sin))
    
        # 计算旋转后的相对位移
        move_x = nW / 2 + np.sin(angle / 180 * np.pi) * d
        move_y = nH / 2 - np.cos(angle / 180 * np.pi) * d
    
        # 调整旋转矩阵的移动距离（t_{x}, t_{y}）
        M[0, 2] += (nW / 2) - cx
        M[1, 2] += (nH / 2) - cy
    
        return cv2.warpAffine(image, M, (nW, nH)), int(move_x), int(move_y)
    
    
    def get_distences(x1, y1, x2, y2):
        return ((x1 - x2) ** 2 + (y1 - y2) ** 2) ** 0.5
    
    def append_img_by_sk_points(img, append_img_path, key_point_y, first_point, second_point, append_img_reset_width=None, append_img_max_height_rate=1, middle_flip=False, append_img_max_height=None):
        ''' 将需要添加的肢体图片进行缩放 '''
        append_image = cv2.imdecode(np.fromfile(append_img_path, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
    
        # 根据长度进行缩放
        sk_height = int(get_distences(first_point[0], first_point[1], second_point[0], second_point[1]) * append_img_max_height_rate)
        # 缩放制约
        if append_img_max_height:
            sk_height = min(sk_height, append_img_max_height)
    
        sk_width = int(
            sk_height / append_image.shape[0] * append_image.shape[1]) if append_img_reset_width is None else int(
            append_img_reset_width)
        if sk_width <= 0:
            sk_width = 1
        if sk_height <= 0:
            sk_height = 1
    
        # 关键点映射
        key_point_y_new = int(key_point_y / append_image.shape[0] * append_image.shape[1])
        # 缩放图片
        append_image = cv2.resize(append_image, (sk_width, sk_height))
    
        img_height, img_width, _ = img.shape
        # 是否根据骨骼节点位置在 图像中间的左右来控制是否进行 左右翻转图片
        # 主要处理头部的翻转, 默认头部是朝左
        if middle_flip:
            middle_x = int(img_width / 2)
            if first_point[0] < middle_x and second_point[0] < middle_x:
                append_image = cv2.flip(append_image, 1)
    
        # 旋转角度
        angle = get_angle(first_point[0], first_point[1], second_point[0], second_point[1])
        append_image, move_x, move_y = rotate_bound(append_image, angle=angle, key_point_y=key_point_y_new)
        app_img_height, app_img_width, _ = append_image.shape
    
        zero_x = first_point[0] - move_x
        zero_y = first_point[1] - move_y
    
        (b, g, r) = cv2.split(append_image)
        for i in range(0, r.shape[0]):
            for j in range(0, r.shape[1]):
                if 230 > r[i][j] > 200 and 0 <= zero_y + i < img_height and 0 <= zero_x + j < img_width:
                    img[zero_y + i][zero_x + j] = append_image[i][j]
        return img
    
    
    body_img_path_map = { 
        "right_hip": "./work/shadow_play_material/right_hip.jpg",
        "right_knee": "./work/shadow_play_material/right_knee.jpg",
        "left_hip": "./work/shadow_play_material/left_hip.jpg",
        "left_knee": "./work/shadow_play_material/left_knee.jpg",
        "left_elbow": "./work/shadow_play_material/left_elbow.jpg",
        "left_wrist": "./work/shadow_play_material/left_wrist.jpg",
        "right_elbow": "./work/shadow_play_material/right_elbow.jpg",
        "right_wrist": "./work/shadow_play_material/right_wrist.jpg",
        "head": "./work/shadow_play_material/head.jpg",
        "body": "./work/shadow_play_material/body.jpg"
    }
    
    
    def get_combine_img(img_path, pose_estimation=pose_estimation, body_img_path_map=body_img_path_map, backgroup_img_path='background/background_01.jpg'):
        ''' 识别图片中的关节点，并将皮影的肢体进行对应，最后与原图像拼接后输出 '''
    
        result = pose_estimation.keypoint_detection(paths=[img_path])
        image = cv2.imread(img_path)
    
        # 背景图片
        backgroup_image = cv2.imread(backgroup_img_path)
        image_flag = cv2.resize(backgroup_image, (image.shape[1], image.shape[0]))
    
        # 最小宽度
        min_width = int(get_distences(result[0]['data']['head_top'][0], result[0]['data']['head_top'][1],
                                      result[0]['data']['upper_neck'][0], result[0]['data']['upper_neck'][1]) / 3)
    
        # 右大腿
        append_img_reset_width = max(int(get_distences(result[0]['data']['pelvis'][0], result[0]['data']['pelvis'][1],
                                                       result[0]['data']['left_hip'][0],
                                                       result[0]['data']['right_hip'][1]) * 1.6), min_width)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['right_hip'], key_point_y=10,
                                             first_point=result[0]['data']['right_hip'],
                                             second_point=result[0]['data']['right_knee'],
                                             append_img_reset_width=append_img_reset_width)
    
        # 右小腿
        append_img_reset_width = max(int(get_distences(result[0]['data']['pelvis'][0], result[0]['data']['pelvis'][1],
                                                       result[0]['data']['left_hip'][0],
                                                       result[0]['data']['right_hip'][1]) * 1.5), min_width)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['right_knee'], key_point_y=10,
                                             first_point=result[0]['data']['right_knee'],
                                             second_point=result[0]['data']['right_ankle'],
                                             append_img_reset_width=append_img_reset_width)
    
        # 左大腿
        append_img_reset_width = max(int(get_distences(result[0]['data']['pelvis'][0], result[0]['data']['pelvis'][1],
                                                       result[0]['data']['left_hip'][0],
                                                       result[0]['data']['left_hip'][1]) * 1.6), min_width)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['left_hip'], key_point_y=0,
                                             first_point=result[0]['data']['left_hip'],
                                             second_point=result[0]['data']['left_knee'],
                                             append_img_reset_width=append_img_reset_width)
    
        # 左小腿
        append_img_reset_width = max(int(get_distences(result[0]['data']['pelvis'][0], result[0]['data']['pelvis'][1],
                                                       result[0]['data']['left_hip'][0],
                                                       result[0]['data']['left_hip'][1]) * 1.5), min_width)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['left_knee'], key_point_y=10,
                                             first_point=result[0]['data']['left_knee'],
                                             second_point=result[0]['data']['left_ankle'],
                                             append_img_reset_width=append_img_reset_width)
    
        # 右手臂
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['left_elbow'], key_point_y=25,
                                             first_point=result[0]['data']['right_shoulder'],
                                             second_point=result[0]['data']['right_elbow'], append_img_max_height_rate=1.2)
    
        # 右手肘
        append_img_max_height = int(get_distences(result[0]['data']['right_shoulder'][0], result[0]['data']['right_shoulder'][1],
                                                  result[0]['data']['right_elbow'][0], result[0]['data']['right_elbow'][1]) * 1.6)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['left_wrist'], key_point_y=10,
                                             first_point=result[0]['data']['right_elbow'],
                                             second_point=result[0]['data']['right_wrist'], append_img_max_height_rate=1.5,
                                             append_img_max_height=append_img_max_height)
    
        # 左手臂
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['right_elbow'], key_point_y=25,
                                             first_point=result[0]['data']['left_shoulder'],
                                             second_point=result[0]['data']['left_elbow'], append_img_max_height_rate=1.2)
    
        # 左手肘
        append_img_max_height = int(get_distences(result[0]['data']['left_shoulder'][0], result[0]['data']['left_shoulder'][1],
                                                  result[0]['data']['left_elbow'][0], result[0]['data']['left_elbow'][1]) * 1.6)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['right_wrist'], key_point_y=10,
                                             first_point=result[0]['data']['left_elbow'],
                                             second_point=result[0]['data']['left_wrist'], append_img_max_height_rate=1.5,
                                             append_img_max_height=append_img_max_height)
    
        # 头
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['head'], key_point_y=10,
                                             first_point=result[0]['data']['head_top'],
                                             second_point=result[0]['data']['upper_neck'], append_img_max_height_rate=1.2,
                                             middle_flip=True)
    
        # 身体
        append_img_reset_width = max(int(get_distences(result[0]['data']['left_shoulder'][0], result[0]['data']['left_shoulder'][1],
                                                       result[0]['data']['right_shoulder'][0], result[0]['data']['right_shoulder'][1]) * 1.2),
                                     min_width * 3)
        image_flag = append_img_by_sk_points(image_flag, body_img_path_map['body'], key_point_y=20,
                                             first_point=result[0]['data']['upper_neck'],
                                             second_point=result[0]['data']['pelvis'],
                                             append_img_reset_width=append_img_reset_width, append_img_max_height_rate=1.2)
    
        result_img = np.concatenate((image, image_flag), axis=1)
    
        return result_img
    
    def transform_video_to_image(video_file_path, img_path):
        ''' 将视频中每一帧保存成图片 '''
        video_capture = cv2.VideoCapture(video_file_path)
        fps = video_capture.get(cv2.CAP_PROP_FPS)
        count = 0
        while(True):
            ret, frame = video_capture.read()
            if ret:
                cv2.imwrite(img_path + '%d.jpg' % count, frame)
                count += 1
            else:
                break
        video_capture.release()
        tkinter.messagebox.showinfo(title='成功', message='视频图片保存成功, 共有 %d 张' % count)
        return fps
    
    # 素材图片位置
    input_video = ''
    
    # 将视频中每一帧保存成图片
    fps = None
    # fps = transform_video_to_image(input_video, 'work/mp4_img/')
    backgroud = ''
    baocunlujin = ''
    
    def analysis_pose(input_frame_path, output_frame_path, is_print=True):
        ''' 分析图片中的人体姿势， 并转换为皮影姿势，输出结果 '''
        global backgroud
    
        file_items = os.listdir(input_frame_path)
        file_len = len(file_items)
        for i, file_item in enumerate(file_items):
            if is_print:
                print(i+1,'/', file_len, ' ', os.path.join(output_frame_path, file_item))
            combine_img = get_combine_img(os.path.join(input_frame_path, file_item))
            cv2.imwrite(os.path.join(output_frame_path, file_item), combine_img)
    
    def combine_image_to_video(comb_path, output_file_path, dds=30, is_print=False):
        ''' 合并图像到视频 '''
        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
    
        file_items = os.listdir(comb_path)
        file_len = len(file_items)
        # print(comb_path, file_items)
        if file_len > 0:
            temp_img = cv2.imread(os.path.join(comb_path, file_items[0]))
            img_height, img_width = temp_img.shape[0], temp_img.shape[1]
            out = cv2.VideoWriter(output_file_path, fourcc, dds, (img_width, img_height))
    
            for i in range(file_len):
                pic_name = os.path.join(comb_path, str(i) + ".jpg")
                if is_print:
                    print(i + 1, '/', file_len, ' ', pic_name)
                img = cv2.imread(pic_name)
                out.write(img)
            out.release()
    
    
    text = ""
    num = 0
    n1 = 0
    
    
    def main():
        def browse_file():
            global folder_file
            filename = filedialog.askopenfilename()
            folder_file.set(filename)
            text1.delete(0, END)
            text1.insert(END, filename)  # 显示在信息框上
    
        def kit():
            global n1
            n1 = n1 + 1
            # print(n1)
            mm = Tk()
            mm.title("提示")
            mm.geometry("300x200")
            messbox = '好吧其实这个功能只是为了占空间的，\n看起来好看一点的。\n给你比心*' + str(n1)
            Label(mm, text=messbox).place(x=50, y=20)
            mm.quit()
            mm.mainloop()
    
        def browse_path():
            global folder_path
            filename = filedialog.askdirectory()
            folder_path.set(filename)
            text2.delete(0, END)
            text2.insert(END, filename)  # 显示在信息框上
    
        def browse_background():
            global folder_background
            filename = filedialog.askopenfilename()
            folder_background.set(filename)
            text3.delete(0, END)
            text3.insert(END, filename)  # 显示在信息框上
    
        def browse_font():
            tkinter.messagebox.showinfo(title='欢迎使用', message='开始下载视频，请耐心等待！')
            time.sleep(3)
            tkinter.messagebox.showinfo(title='完成', message='视频已下载完成，请查看！')
            os.system("start ./download/下载视频.mp4")
    
        def browse_pic():
            tkinter.messagebox.showinfo(title='欢迎使用', message='浏览模式已启动，请耐心等待！')
            os.system("start ./成品/mp4_analysis.mp4")
    
        def helps():
            os.system("start ./mess/helps.txt")
    
        def mins():
            os.system("shutdown -p")
    
        def start():
            e13 = text1.get()   # 选择文件
            e14 = text2.get()   # 保存路径
            e15 = text3.get()   # 背景图片
            e16 = text4.get()   # 网址下载
            if e13 == "":
                tkinter.messagebox.showwarning(title='空输入！', message='您尚未选择目标文件')  # 提出警告对话窗
            else:
                tkinter.messagebox.showinfo(title='欢迎使用', message='正在转换中，请关闭此弹窗后耐心等待！')  # 提示信息对话窗
    
                # 背景图片显示
                global backgroud
                background=Image.open(e15)
                backgroud = e15
    
                # font=ImageFont.truetype(e16)
    
                # 素材图片位置
                global input_video
                input_video = e13
    
                global fps
                fps = transform_video_to_image(input_video, 'work/mp4_img/')
                # 分析图片中的人体姿势， 并转换为皮影姿势，输出结果
                analysis_pose('work/mp4_img/', 'work/mp4_img_analysis/', is_print=False)
    
                # 合并图像到视频
                global baocunlujin
                baocunlujin = e14 + '/mp4_analysis.mp4'
                combine_image_to_video('work/mp4_img_analysis/',baocunlujin, fps)
                # print('成功')
                tkinter.messagebox.showinfo(title='成功', message='转换完毕！')
    
        tk = Tk()
    
        tk.title("皮影戏生成器V2.0")
    
        # 菜单
        menubar = Menu(tk)
        filemenu = Menu(menubar, tearoff=False)
        filemenu.add_command(label="粘贴", command=kit)
        filemenu.add_command(label="复制", command=kit)
        menubar.add_cascade(label="文件", menu=filemenu)
    
        filemenu1 = Menu(menubar, tearoff=False)
        filemenu1.add_command(label="立即关机", command=mins)
        menubar.add_cascade(label="关机", menu=filemenu1)
    
        filemenu2 = Menu(menubar, tearoff=False)
        filemenu2.add_command(label="帮助", command=helps)
        menubar.add_cascade(label="帮助", menu=filemenu2)
        tk.config(menu=menubar)
        # 创建frame容器
    
        frameLT = Frame(width=300, height=500)  # 左容器
    
        frameRT = Frame(width=330, height=500)  # 右容器
    
        global folder_file
        folder_file = StringVar()
        global folder_path
        folder_path = StringVar()
        global folder_background
        folder_background = StringVar()
        global folder_font
        folder_font = StringVar()
    
        #
        # 图片显示大小更改与适应框架
        # global pt
    
        # 效果预览
    
        img = Image.open('./example/样品.jpg')
        photo = ImageTk.PhotoImage(img.resize((330, 500)))
    
        # 效果预览
        label = Label(frameRT, text="软件工程",  # 设置文本内容
                      justify='left',  # 设置文本对齐方式：左对齐
                      anchor='center',  # 设置文本在label的方位：中心方位
                      font=('微软雅黑', 18),  # 设置字体：微软雅黑，字号：18
                      fg='red',  # 设置前景色：白色
                      image=photo,  # 设置图像
                      compound='center',  # 设置文本与图像结合方式
                      )
        label.grid()
        # 创建控件
        # frameRT.update()
        # pt='./example/bk1_李国夫手写体.png'
        text1 = Entry(frameLT)
        button1 = Button(text="选择文件", command=browse_file, height=1, width=6)
    
        text2 = Entry(frameLT)
        text2.insert(END, './成品')
        button2 = Button(text="保存路径", command=browse_path, height=1, width=6)
    
        text3 = Entry(frameLT)
        text3.insert(END, "./background/background_01.jpg")
    
        button3 = Button(text="选择背景", command=browse_background, height=1, width=6)
    
        text4 = Entry(frameLT)
        text4.insert(END, '需要下载的视频网址放在这里')
    
        button4 = Button(text="网址下载", command=browse_font, height=1, width=6)
    
        button6 = Button(text="浏览", command=browse_pic, height=2, width=12)
    
        button7 = Button(text="开始", command=start, height=2, width=12)
    
    
        Label(frameLT, text="祝王老师万事如意，Happy life！！！").place(x=5, y=470)
    
        Label(frameLT, text="简易使用说明：\n1.点击'选择文件'按键，选择需要操作的原视频文件。\n2.'保存路径'按键用来选择生成文件所保存的位置。\n3.选择背景可以用于选择生成的表演背景。\n4.网址下载处可以放置视频链接然后点击下载。\n5.点击'开始'按键进行生成。\n6.点击'浏览'按钮进行浏览完成视频.").place(x=13, y=160)
        # 窗体布局
    
        frameLT.grid(row=0, column=0, columnspan=2, padx=1, pady=5)
    
        frameRT.grid(row=0, column=2, rowspan=3, padx=5, pady=4)
    
        # 固定大小
    
        frameLT.grid_propagate(0)
    
        frameRT.grid_propagate(0)
    
        # 控件布局
        text1.place(x=20, y=20, height=20, width=210)
        button1.place(x=240, y=20)
        text2.place(x=20, y=50, height=20, width=210)
        button2.place(x=240, y=50)
        text3.place(x=20, y=80, height=20, width=210)
        button3.place(x=240, y=80)
        text4.place(x=20, y=110, height=20, width=210)
        button4.place(x=240, y=110)
        button6.place(x=30, y=410)
        button7.place(x=170, y=410)
    
        # 主事件循环
    
        tk.mainloop()
    
    
    main()

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjAyMDc4OA_size_16_color_FFFFFF_t_70]: /images/20221005/c831f024dd084e92a4c88c0cec959a52.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NjAyMDc4OA_size_16_color_FFFFFF_t_70 1]: /images/20221005/de352f37f4f24a7098a01aedeb4a572c.png