分布式系统删除zookeeper空节点

小灰灰 2022-01-23 23:19 341阅读 0赞

在分布式系统结构中，使用ZOOKEEPER里的CuratorFramework控制全局锁，有一个隐藏的问题，生成的全局锁路径释放后，存留了很多空的NODE节点，这些节点数据量很大的时候，会占用很大的磁盘空间，这个可能是ZK版本的一个bug，要是使用完全局锁之后自动帮我们清除掉，就不会有这类问题，针对bug，需要定时清掉空的节点。代码如下，以供记录一下。

1、测试类，测试定时任务手工执行接口，执行成功后，查看一下节点是否真实删除

package xxxxxxxxxxxxxxxxx
    
    import xxxxxxxxxxxx
    import io.swagger.annotations.Api;
    import io.swagger.annotations.ApiOperation;
    import lombok.extern.slf4j.Slf4j;
    import org.apache.curator.RetryPolicy;
    import org.apache.curator.framework.CuratorFramework;
    import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
    import org.apache.curator.retry.ExponentialBackoffRetry;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.validation.BindingResult;
    import org.springframework.web.bind.annotation.*;
    
    import javax.validation.Valid;
    import java.util.HashMap;
    import java.util.Map;
    
    @RestController
    @Slf4j
    @Api(value="test", description="测试")
    public class TestController {
    
    	private final CuratorFramework curator;
    	private final LockNodeCleanService lockNodeCleanService;
    
    
    
    	public TestController(CuratorFramework curator,LockNodeCleanService lockNodeCleanService) {
    		this.curator = curator;
    		this.lockNodeCleanService =lockNodeCleanService;
    	}
    
    
    	@GetMapping("/internal/zookeeper/listData")
    	public void listData(@RequestParam String zkPath) throws Exception {
    //		RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
    
    //		Map<String, Long> result = new HashMap<>();
    		for (String string : curator.getChildren().forPath(zkPath)) {
    			String tempPath = zkPath + "/" + string;
    			log.info("***********:"+tempPath);
    		}
    	}
    
    	@GetMapping("/internal/zookeeper/remove")
    	public void listData() throws Exception {
    		lockNodeCleanService.deleteEmptyLockNodes();
    	}
    
    }

2、定时任务Servic，自己服务中的节点同时配置了4级和5级两种节点，大家自己服务最好能够统一一种，这样方便处理哈哈。

package xxxxxxxxxxxxxx
    
    import lombok.AllArgsConstructor;
    import lombok.extern.slf4j.Slf4j;
    import org.apache.commons.collections.CollectionUtils;
    import org.apache.curator.framework.CuratorFramework;
    import org.apache.curator.framework.CuratorFrameworkFactory;
    import org.apache.curator.framework.recipes.leader.LeaderLatch;
    import org.apache.curator.framework.recipes.locks.InterProcessMutex;
    import org.apache.curator.retry.RetryNTimes;
    import org.apache.zookeeper.data.Stat;
    import org.springframework.scheduling.annotation.Scheduled;
    import org.springframework.stereotype.Service;
    
    import java.util.List;
    
    @Service
    @Slf4j
    @AllArgsConstructor
    public class LockNodeCleanService {
        private final CuratorFramework curator;
        private final LeaderLatch leaderLatch;
        private static final String root = "/locks/quality-test";
    
    
        public static void main(String[] args) throws Exception {
            CuratorFramework curator = curator(args[0]);
            InterProcessMutex lock = new InterProcessMutex(curator, "");
            lock.acquire();
            lock.release();
            LockNodeCleanService service = new LockNodeCleanService(curator, null);
            service.deleteEmptyLockNodes();
        }
    
        private static CuratorFramework curator(String zkAddress) {
            CuratorFramework curator = CuratorFrameworkFactory.
                    builder().
                    connectString(zkAddress).
                    sessionTimeoutMs(1000).
                    retryPolicy(new RetryNTimes(3, 1000)).
                    build();
            curator.start();
            return curator;
        }
    
        /**
         * 删除所有空的节点
         */
        @Scheduled(cron = "0 0 4 * * ?")
        public void deleteEmptyLockNodes() {
            if (!leaderLatch.hasLeadership()) {
                log.info("deleteEmptyLockNodes not leader");
                return;
            }
            try {
                //二级目录是否存在
                Stat stat = curator.checkExists().forPath(root);
                if (stat == null) {
                    return;
                }
    
                List<String> folders = curator.getChildren().forPath(root);
                if (CollectionUtils.isEmpty(folders)) {
                    return;
                }
                for (String folder : folders) {
                    //三级路径
                    String folderFullPath = root + "/" + folder;
                    try {
                        clearOneFolder(folderFullPath);
                    } catch (Exception e) {
                        log.info("error when clear folder: {}", folderFullPath, e);
                    }
                }
            } catch (Exception e) {
                log.error("x", e);
    
            }
        }
    
    
        /**
         * 删除一个文件夹下的空节点
         * 一个文件夹就是一种业务对象
         *
         * @param folder
         * @throws Exception
         */
        private void clearOneFolder(String folder) throws Exception {
            for (String nodePath : curator.getChildren().forPath(folder)) {
                //四级路径
                String nodeFullPath = folder + "/" + nodePath;
                List<String> folders = curator.getChildren().forPath(nodeFullPath);
                if (CollectionUtils.isEmpty(folders)) {
                    try {
                        //没有子目录直接删除
                        deleteNode(nodeFullPath, curator);
                    } catch (Exception e) {
                        log.info("fail to delete: {}", nodeFullPath, e);
                    }
                    continue;
                }
                for (String path : folders) {
                    //删除第五级目录
                    String pathFive = nodeFullPath + "/" + path;
                    try {
                        deleteNode(pathFive, curator);
                    } catch (Exception e) {
                        log.info("fail to delete: {}", pathFive, e);
                    }
                }
                try {//删除第四级目录
                    deleteNode(nodeFullPath, curator);
                } catch (Exception e) {
                    log.info("fail to delete: {}", nodeFullPath, e);
                }
    
    
    
    
            }
        }
    
    
        /**
         * 安全删除由于锁造成的空节点
         * 确保节点存在、没有子节点、没有数据、最近修改时间在一小时前
         *
         * @param path
         * @param curator
         * @throws Exception
         */
        public static void deleteNode(String path, CuratorFramework curator) throws Exception {
            Stat stat = curator.checkExists().forPath(path);
            if (stat == null) {
                log.info(path + "  not exist");
                return;
            }
            if (stat.getNumChildren() > 0) {
                log.info(path + "  has children");
                return;
            }
            if (curator.getData().forPath(path).length > 0) {
                log.info(path + "  has data");
                return;
            }
            if (System.currentTimeMillis() - stat.getMtime() < 3600 * 1000) {
                log.info(path + "  very new");
                return;
            }
            curator.delete().inBackground().forPath(path);
            log.info(path + "  is deleted");
        }
    
    }