springboot连接操作集群的hbase

ゝ一世哀愁。 2022-02-16 11:53 1073阅读 0赞

**服务器环境：**hadoop2.7.3集群、zookeper3.4.10集群、hbase1.2.5集群

**测试环境：**windows10、spring tool suite(STS)

**一、测试前准备**

1、由于是在windows连接Hbase，而hbase是依赖于hadoop的，所以我们需要下载hadoop的二进制包，这里我仅提供2.7.3版本的。下载地址：[https://pan.baidu.com/s/1eL6fEU9AFMfZ871xnlJ0-g][https_pan.baidu.com_s_1eL6fEU9AFMfZ871xnlJ0-g]  提取码：vwel

把下载的haddop二进制包解压到你磁盘下的某个目录，后面配置需要用到。

2、本测试使用的是maven管理JAR包，还没安装maven的可以参考：[Maven工程项目的创建（Eclipse）、打包及运行（Windows cmd）][Maven_Eclipse_Windows cmd]

3、打开本地 C:\\Windows\\System32\\drivers\\etc下的名为hosts的系统文件。在文件底部插入集群名映射到ip，参考下图：

![20190422180150694.png][]

**二、添加依赖**

在pom.xml文件的<dependencies></dependencies>中添加以下依赖：

<dependency>
    			<groupId>org.apache.hbase</groupId>
    			<artifactId>hbase-client</artifactId>
    			<version>1.2.5</version>
    			<exclusions>
    				<exclusion>
    					<groupId>org.slf4j</groupId>
    					<artifactId>slf4j-log4j12</artifactId>
    				</exclusion>
    			</exclusions>
    		</dependency>
    		<dependency>
    			<groupId>org.springframework.data</groupId>
    			<artifactId>spring-data-hadoop</artifactId>
    			<version>2.5.0.RELEASE</version>
    		</dependency>
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-hdfs</artifactId>
    			<version>2.7.3</version>
    		</dependency>
    		<dependency>
    			<groupId>org.springframework.data</groupId>
    			<artifactId>spring-data-hadoop-core</artifactId>
    			<version>2.4.0.RELEASE</version>
    		</dependency>
    		<dependency>
    			<groupId>org.apache.hbase</groupId>
    			<artifactId>hbase</artifactId>
    			<version>1.2.5</version>
    			<type>pom</type>
    		</dependency>

**二、新建HBase工具类**

新建一个HBaseUtils工具类，代码如下：

HBaseUtils.java：

package com.qcxy.helper.hbase;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.hbase.HBaseConfiguration;
    import org.apache.hadoop.hbase.TableName;
    import org.apache.hadoop.hbase.client.*;
    import java.io.IOException;
    import java.util.ArrayList;
    import java.util.List;
    import java.util.Map;
    import java.util.Properties;
    
    /**
     * HBase工具类 Author zzq
     */
    public class HBaseUtils {
    	private static Connection connection;
    	private static Configuration configuration;
    	private static HBaseUtils hBaseUtils;
    	private static Properties properties;
    
    	/**
    	 * 创建连接池并初始化环境配置
    	 */
    	public void init() {
    		properties = System.getProperties();
    		// 实例化HBase配置类
    		if (configuration == null) {
    			configuration = HBaseConfiguration.create();
    		}
    		try {
    			// 加载本地hadoop二进制包，换成你解压的地址
    			properties.setProperty("hadoop.home.dir", "E:\\springboot\\hadoop-common-2.7.3-bin-master");
    			// zookeeper集群的URL配置信息
    			configuration.set("hbase.zookeeper.quorum", "master,slave01,slave02");
    			// HBase的Master
    			configuration.set("hbase.master", "master:90000");
    			// 客户端连接zookeeper端口
    			configuration.set("hbase.zookeeper.property.clientPort", "2181");
    			// HBase RPC请求超时时间，默认60s(60000)
    			configuration.setInt("hbase.rpc.timeout", 20000);
    			// 客户端重试最大次数，默认35
    			configuration.setInt("hbase.client.retries.number", 10);
    			// 客户端发起一次操作数据请求直至得到响应之间的总超时时间，可能包含多个RPC请求，默认为2min
    			configuration.setInt("hbase.client.operation.timeout", 30000);
    			// 客户端发起一次scan操作的rpc调用至得到响应之间的总超时时间
    			configuration.setInt("hbase.client.scanner.timeout.period", 200000);
    			// 获取hbase连接对象
    			if (connection == null || connection.isClosed()) {
    				connection = ConnectionFactory.createConnection(configuration);
    			}
    		} catch (IOException e) {
    			e.printStackTrace();
    		}
    	}
    
    	/**
    	 * 关闭连接池
    	 */
    	public static void close() {
    		try {
    			if (connection != null)
    				connection.close();
    		} catch (IOException e) {
    			e.printStackTrace();
    		}
    	}
    
    	/**
    	 * 私有无参构造方法
    	 */
    	private HBaseUtils() {
    	}
    
    	/**
    	 * 唯一实例，线程安全，保证连接池唯一
    	 * 
    	 * @return
    	 */
    	public static HBaseUtils getInstance() {
    		if (hBaseUtils == null) {
    			synchronized (HBaseUtils.class) {
    				if (hBaseUtils == null) {
    					hBaseUtils = new HBaseUtils();
    					hBaseUtils.init();
    				}
    			}
    		}
    		return hBaseUtils;
    	}
    
    	/**
    	 * 单行插入数据
    	 * 
    	 * @param tablename
    	 * @param rowkey
    	 * @param family
    	 * @param cloumns
    	 * @throws IOException
    	 */
    	public static void put(String tablename, String rowkey, String family, Map<String, String> cloumns)
    			throws IOException {
    		Table table = null;
    		try {
    			table = connection.getTable(TableName.valueOf(tablename));
    			Put put = new Put(rowkey.getBytes());
    			for (Map.Entry<String, String> entry : cloumns.entrySet()) {
    				put.addColumn(family.getBytes(), entry.getKey().getBytes(), entry.getValue().getBytes());
    			}
    			table.put(put);
    		} catch (IOException e) {
    			e.printStackTrace();
    		} finally {
    			table.close();
    			close();
    		}
    	}
    
    	/**
    	 * 获取单条数据
    	 * 
    	 * @param tablename
    	 * @param row
    	 * @return
    	 * @throws IOException
    	 */
    	public static Result getRow(String tablename, String row) throws IOException {
    		Table table = null;
    		Result result = null;
    		try {
    			table = connection.getTable(TableName.valueOf(tablename));
    			Get get = new Get(row.getBytes());
    			result = table.get(get);
    		} finally {
    			table.close();
    		}
    		return result;
    	}
    
    	/**
    	 * 查询多行信息
    	 * 
    	 * @param tablename
    	 * @param rows
    	 * @return
    	 * @throws IOException
    	 */
    	public static Result[] getRows(String tablename, List<byte[]> rows) throws IOException {
    		Table table = null;
    		List<Get> gets = null;
    		Result[] results = null;
    		try {
    			table = connection.getTable(TableName.valueOf(tablename));
    			gets = new ArrayList<Get>();
    			for (byte[] row : rows) {
    				if (row != null) {
    					gets.add(new Get(row));
    				}
    			}
    			if (gets.size() > 0) {
    				results = table.get(gets);
    			}
    		} catch (IOException e) {
    			e.printStackTrace();
    		} finally {
    			table.close();
    		}
    		return results;
    	}
    
    	/**
    	 * 获取整表数据
    	 * 
    	 * @param tablename
    	 * @return
    	 */
    	public static ResultScanner get(String tablename) throws IOException {
    		Table table = null;
    		ResultScanner results = null;
    		try {
    			table = connection.getTable(TableName.valueOf(tablename));
    			Scan scan = new Scan();
    			scan.setCaching(1000);
    			results = table.getScanner(scan);
    		} catch (IOException e) {
    			e.printStackTrace();
    		} finally {
    			table.close();
    		}
    		return results;
    	}
    
    	/**
    	 * 删除数据
    	 * 
    	 * @param tablename
    	 * @param family
    	 * @param column
    	 * @param row
    	 * @throws IOException
    	 */
    	public static void delete(String tablename, String family, String column, String row) throws IOException {
    		Table table = null;
    
    		try {
    			table = connection.getTable(TableName.valueOf(tablename));
    			Delete del = new Delete(row.getBytes());
    			del.addColumns(family.getBytes(), column.getBytes());
    			table.delete(del);
    		} finally {
    			table.close();
    		}
    
    	}
    
    }

**三、操作Hbase**

**1、建表**

在服务器中切换到hbase安装目录下的bin目录执行如下命令，使用hbase shell

$ hbase shell

在hbase shell 模式下我们执行以下命令新建一张表,在建表时要有列族，以下创建了张名为person的表，并且这张表中有一个名为info的列族。

create 'person','info'

执行命令成功后你会看到以下信息：

![20190422182539259.PNG][]

我们也可以用list命令验证下有没有此表

![20190422182702465.PNG][]

可以看到是有此表的。 我们建了结构如下的一张表

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6cWFhYXNzcw_size_16_color_FFFFFF_t_70][]

**2、增加数据**

public void putdata() {
                    //获取连接池唯
    		HBaseUtils.getInstance();
    		try {
    			Map<String, String> cloumns = new HashMap<String, String>();
    			cloumns.put("name", "zzq");    //列名和值
    			cloumns.put("age", "22");    //列名和值
                            //往表中的第1row的info族中增加了名为name、age的列并给出数据
    			HBaseUtils.put("person", "1", "info", cloumns);
    			System.out.println("增加成功");
    		} catch (IOException e) {
    			e.printStackTrace();
    			System.out.println("增加失败");
    		} finally {
    			HBaseUtils.close();
    		}
    	}

执行此函数后我们的表如下

![20190422184038805.PNG][]

**3、更新数据**

更新数据和增加数据一样，更改值后执行以上代码它会覆盖原来的值

**4、获取数据**

public void getTableAllData() {
    		HBaseUtils.getInstance();
    		ResultScanner results = null;
    		try {
    			results = HBaseUtils.get("person");//表名
    
    			for (Result result : results) {
    				NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> navigableMap = result.getMap();
    				for (byte[] family : navigableMap.keySet()) {
    					System.out.println("列族:" + new String(family));
    					for (byte[] column : navigableMap.get(family).keySet()) {
    						System.out.println("列:" + new String(column));
    						for (Long t : navigableMap.get(family).get(column).keySet()) {
    							System.out.println("值:" + new String(navigableMap.get(family).get(column).get(t)));
    						}
    					}
    				}
    			}
    		} catch (IOException e) {
    			e.printStackTrace();
    		} finally {
    			results.close();
    			HBaseUtils.close();
    		}
    	}

执行成功后会输出如下信息：

![20190422184629932.PNG][]

**5、删除数据**

public void delete() {
    		HBaseUtils.getInstance();
    		try {   //删除person表中的第1行的info列族中的name列中的的值
    			HBaseUtils.delete("person", "info", "name", "1");
    			System.out.println("删除成功");
    		} catch (IOException e) {
    			System.out.println("删除失败");
    			e.printStackTrace();
    		}
    	}

删除成功后我们在查询表中的数据如下

![20190422184917863.PNG][]

以上操作代码的整合

Test.java：

package com.qcxy.helper.Controller;
    
    import org.springframework.stereotype.Controller;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.ResponseBody;
    import org.apache.hadoop.hbase.client.Result;
    import org.apache.hadoop.hbase.client.ResultScanner;
    import java.io.IOException;
    import java.util.*;
    import com.qcxy.helper.hbase.HBaseUtils;
    
    @Controller
    public class Test {
    
    	@ResponseBody
    	@RequestMapping("/hello")
    	public String hello() {
    		return "hello";
    	}
    
    	/**
    	 * 增加数据
    	 */
    	@ResponseBody
    	@RequestMapping("/putdata")
    	public void putdata() {
    		HBaseUtils.getInstance();
    		try {
    			Map<String, String> cloumns = new HashMap<String, String>();
    			cloumns.put("name", "zzq");
    			cloumns.put("age", "22");
    			HBaseUtils.put("person", "1", "info", cloumns);
    			System.out.println("增加成功");
    		} catch (IOException e) {
    			e.printStackTrace();
    			System.out.println("增加失败");
    		} finally {
    			HBaseUtils.close();
    		}
    	}
    
    	/**
    	 * 获取整表数据
    	 */
    	@ResponseBody
    	@RequestMapping("/getTableAllData")
    	public void getTableAllData() {
    		HBaseUtils.getInstance();
    		ResultScanner results = null;
    		try {
    			results = HBaseUtils.get("person");
    
    			for (Result result : results) {
    				NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> navigableMap = result.getMap();
    				for (byte[] family : navigableMap.keySet()) {
    					System.out.println("列族:" + new String(family));
    					for (byte[] column : navigableMap.get(family).keySet()) {
    						System.out.println("列:" + new String(column));
    						for (Long t : navigableMap.get(family).get(column).keySet()) {
    							System.out.println("值:" + new String(navigableMap.get(family).get(column).get(t)));
    						}
    					}
    				}
    			}
    		} catch (IOException e) {
    			e.printStackTrace();
    		} finally {
    			results.close();
    			HBaseUtils.close();
    		}
    	}
    
    	/**
    	 * 删除数据
    	 */
    	@ResponseBody
    	@RequestMapping("/deletedata")
    	public void delete() {
    		HBaseUtils.getInstance();
    		try {
    			HBaseUtils.delete("person", "info", "name", "1");
    			System.out.println("删除成功");
    		} catch (IOException e) {
    			System.out.println("删除失败");
    			e.printStackTrace();
    		}
    	}
    
    }

到此本次测试完成！

[https_pan.baidu.com_s_1eL6fEU9AFMfZ871xnlJ0-g]: https://pan.baidu.com/s/1eL6fEU9AFMfZ871xnlJ0-g
[Maven_Eclipse_Windows cmd]: https://blog.csdn.net/qq_42881421/article/details/82945454
[20190422180150694.png]: /images/20220216/e504c470072c4fb6abf9cb1cc576f350.png
[20190422182539259.PNG]: /images/20220216/1c27d4b913634caaa44e596be5ad0045.png
[20190422182702465.PNG]: /images/20220216/111709cb4f784876b1aed72e548ff699.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p6cWFhYXNzcw_size_16_color_FFFFFF_t_70]: /images/20220216/92ea13aa90fe44cf913807136f33121c.png
[20190422184038805.PNG]: /images/20220216/ef0ef9e5044c4a37bd1edaa313a3f6f9.png
[20190422184629932.PNG]: /images/20220216/f211399e48934cc7be4ed5b6d2f7e1ad.png
[20190422184917863.PNG]: /images/20220216/00c2e6bfd9b04d4ab8a0837112dd3195.png