Minio大文件分片上传、断点续传实现

拼搏现实的明天。 2023-09-30 19:48 42阅读 0赞

使用minio api实现分片上传及断点续传功能。  
前端准备：获取大文件的MD5值，将文件分片，5M为一分片，排好顺序，并按顺序命名（1,2,3这种后面比较好合并）

![在这里插入图片描述][3c91c515b2a04306b24be7f6ba1084ee.png]

在上传分片阶段，前端有上传进度条  
![在这里插入图片描述][watermark_type_d3F5LXplbmhlaQ_shadow_50_text_Q1NETiBA5oiR5piv5LiA5Y-q5bCP6Z2S6JuZ5ZGx5ZGx5ZGx5ZGx5ZGx_size_10_color_FFFFFF_t_70_g_se_x_16]  
1、检验文件MD5值  
1.1 redis中查看MD5是否存在

String data;
            try {
        
                data = customRedisCache.getCacheObject(md5);
            } catch (Exception e) {
        
                e.printStackTrace();
            }

1.2 判断临时文件夹是否存在  
boolean d = doesFolderExist(bucket, md5);

/**
         * 判断文件夹是否存在
         *
         * @param bucketName 存储桶
         * @param objectName 文件夹名称（去掉/）
         * @return true：存在
         */
        public boolean doesFolderExist(String bucketName, String objectName) {
        
            boolean exist = false;
            try {
        
                Iterable<Result<Item>> results = minioClient.listObjects(
                        ListObjectsArgs.builder().bucket(bucketName).prefix(objectName).recursive(false).build());
                StringBuilder objectNameBuilder = new StringBuilder(objectName);
                for (Result<Item> result : results) {
        
                    objectNameBuilder.append("/");
                    Item item = result.get();
                    if (item.isDir() && objectNameBuilder.toString().equals(item.objectName())) {
        
                        exist = true;
                    }
                }
            } catch (Exception e) {
        
                exist = false;
            }
            return exist;
        }

1.3 删除临时文件夹及其中文件  
deleteObject(bucket, md5);

public void deleteObject(String bucketName, String objectName) {
        
            String objectNames = objectName + "/";
            try {
        
                if (StringUtils.isNotBlank(objectNames)) {
        
                    if (objectNames.endsWith(".") || objectNames.endsWith("/")) {
        
                        Iterable<Result<Item>> list = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).prefix(objectNames).recursive(false).build());
                        list.forEach(e -> {
        
                            try {
        
                                minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName).object(e.get().objectName()).build());
                            } catch (Exception a) {
        
                                return;
                            }
    
                        });
                    }
                }
            } catch (Exception e) {
        
                e.printStackTrace();
            }
        }

1.4 上传文件和上传分片一样

InputStream inputStream;
            try {
        
                inputStream = file.getInputStream();
                minioClient.putObject(PutObjectArgs.builder()
                        .bucket(packages)
                        .object(md5 + "/" + chunk)
                        .stream(inputStream, file.getSize(), -1)
                        .contentType("application/x-img")
                        .build());
            } catch (Exception e) {
        
                e.printStackTrace();
                return new ResultDto(ResultDto.ERROR, "上传失败", chunk);
            }
            return new ResultDto(ResultDto.SUCCESS, "上传成功", chunk);

.object(md5 + “/” + chunk) ：这种方式在minio里会先创建文件夹

1.5 合并分片  
boolean b = merge(bucket, md5, fileName);

/**
         * 将块文件合并到新桶   块文件必须满足 名字是 0 1  2 3 5....
         *
         * @param bucketName 存块文件的桶
         * @param objectName 存新文件的桶
         * @param fileName1  存到新桶中的文件名称
         * @return
         */
        public boolean merge(String bucketName, String objectName, String fileName1) {
        
            try {
        
                List<ComposeSource> sourceObjectList = new ArrayList<ComposeSource>();
                List<Object> folderList = this.getFolderList(bucketName, objectName);
                List<String> fileNames = new ArrayList<>();
                if (folderList != null && !folderList.isEmpty()) {
        
                    for (Object value : folderList) {
        
                        Map o = (Map) value;
                        String name = (String) o.get("fileName");
                        fileNames.add(name);
                    }
                }
                List<Integer> fileNameInt = new ArrayList<>();
                List<String> fileNameLast = new ArrayList<>();
                if (!fileNames.isEmpty()) {
        
                    for (String fileName : fileNames) {
        
                        fileNameInt.add(Integer.parseInt(fileName.split("/")[1]));
                    }
                    Collections.sort(fileNameInt);
                    for (int j = 0; j < fileNameInt.size(); j++) {
        
                        fileNameLast.add(fileNames.get(j).split("/")[0] + "/" + fileNameInt.get(j));
                    }
                    for (String name : fileNameLast) {
        
                        sourceObjectList.add(ComposeSource.builder().bucket(bucketName).object(name).build());
                    }
                }
    
                minioClient.composeObject(
                        ComposeObjectArgs.builder()
                                .bucket(bucketName)
                                .object(fileName1)
                                .sources(sourceObjectList)
                                .build());
                return true;
            } catch (Exception e) {
        
                e.printStackTrace();
            }
            return false;
        }

//获取指定bucketName下所有文件 文件名+大小
        public List<Object> getFolderList(String bucketName, String objectName) throws Exception {
        
            String objectNames = objectName + "/";
            Iterable<Result<Item>> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName).prefix(objectNames).recursive(false).build());
            Iterator<Result<Item>> iterator = results.iterator();
            List<Object> items = new ArrayList<>();
            String format = "{'fileName':'%s','fileSize':'%s'}";
            while (iterator.hasNext()) {
        
                Item item = iterator.next().get();
                items.add(JSON.parse((String.format(format, item.objectName(),
                        formatFileSize(item.size())))));
            }
            return items;
        }

private static String formatFileSize(long fileS) {
        
            DecimalFormat df = new DecimalFormat("#.00");
            String fileSizeString;
            String wrongSize = "0B";
            if (fileS == 0) {
        
                return wrongSize;
            }
            if (fileS < 1024) {
        
                fileSizeString = df.format((double) fileS) + " B";
            } else if (fileS < 1048576) {
        
                fileSizeString = df.format((double) fileS / 1024) + " KB";
            } else if (fileS < 1073741824) {
        
                fileSizeString = df.format((double) fileS / 1048576) + " MB";
            } else {
        
                fileSizeString = df.format((double) fileS / 1073741824) + " GB";
            }
            return fileSizeString;
        }

1.6 检验MD5值  
boolean checkMd5 = checkMd5(bucket, fileName, md5);

private boolean checkMd5(String bucketName, String fileName, String md5) {
        
            try {
        
                //利用apache工具类获取文件md5值
                InputStream inputStream = getInput(bucketName, fileName);
                String md5Hex = DigestUtils.md5Hex(inputStream);
                if (md5.equalsIgnoreCase(md5Hex)) {
        
                    return true;
                }
            } catch (Exception e) {
        
                e.printStackTrace();
            }
            return false;
        }

[3c91c515b2a04306b24be7f6ba1084ee.png]: https://img-blog.csdnimg.cn/3c91c515b2a04306b24be7f6ba1084ee.png
[watermark_type_d3F5LXplbmhlaQ_shadow_50_text_Q1NETiBA5oiR5piv5LiA5Y-q5bCP6Z2S6JuZ5ZGx5ZGx5ZGx5ZGx5ZGx_size_10_color_FFFFFF_t_70_g_se_x_16]: https://img-blog.csdnimg.cn/d6f6618d05024f9782d034cafd9ade72.png?x-oss-process=image/watermark,type_d3F5LXplbmhlaQ,shadow_50,text_Q1NETiBA5oiR5piv5LiA5Y-q5bCP6Z2S6JuZ5ZGx5ZGx5ZGx5ZGx5ZGx,size_10,color_FFFFFF,t_70,g_se,x_16