Hbase使用javaAPI链接调用和过滤器API的使用

忘是亡心i 2022-03-30 08:22 223阅读 0赞

## javaAPI调用链接Hbase ##

javaAPI链接Hbase的基础demo,后面的过滤部分也以此table为主

pom

<dependency>
    			<groupId>org.apache.hbase</groupId>
    			<artifactId>hbase-client</artifactId>
    			<version>1.4.9</version>
    		</dependency>
    		<dependency>
    			<groupId>org.apache.hbase</groupId>
    			<artifactId>hbase-server</artifactId>
    			<version>1.4.9</version>
    		</dependency>
    		<dependency>
    		<groupId>org.apache.hbase</groupId>
    		<artifactId>hbase-common</artifactId>
    		<version>1.4.9</version>
    		</dependency>

java

public class HbaseDemo {
    
    
        private static Admin admin;
    
        public static void main(String[] args){
            System.setProperty("hadoop.home.dir", "d:/");
    
            String tableName ="user_table";
            try {
                createTable(tableName, new String[] { "information", "contact" });
                showTable(tableName);
                User user = new User("001", "xiaoming", "123456", "man", "20", "13355550021", "1232821@bmaoo.com");
                insertData("user_table", user);
                User user2 = new User("002", "xiaohong", "654321", "female", "18", "18757912212", "214214@bmaoo.com");
                insertData(tableName, user2);
    
                System.out.println("--------------------获取原始数据-----------------------");
                getNoDealData(tableName);
                System.out.println("--------------------行列簇值对应关系数据-----------------------");
                getAllRows(tableName);
                List<User> list = getAllData(tableName);
                System.out.println("--------------------原始数据解析处理后--------------------");
                for (User user3 : list){
                    System.out.println(user3.toString());
                }
    
                System.out.println("--------------------根据rowKey查询--------------------");
                User user4 = getDataByRowKey("user_table", "user-001");
                System.out.println(user4.toString());
                System.out.println("--------------------获取指定列族:列的属性值数据-------------------");
                String user_phone = getCellData("user_table", "user-001", "contact", "phone");
                System.out.println(user_phone);
                System.out.println("--------------------插入测试数据--------------------");
    //            User user5 = new User("test-003", "xiaoguang", "789012", "man", "22", "12312132214", "856832@bmaoo.com");
    //            insertData("user_table", user5);
    //            List<User> list2 = getAllData("user_table");
    //
    //            for (User user6 : list2){
    //                System.out.println(user6.toString());
    //            }
    
    //            System.out.println("--------------------删除测试数据--------------------");
    //            deleteByRowKey("user_table", "user-test-003");
    //            List<User> list3 = getAllData("user_table");
    //
    //            for (User user7 : list3){
    //                System.out.println(user7.toString());
    //            }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    
        //连接集群
        public static Connection initHbase() throws IOException {
    
            Configuration configuration = HBaseConfiguration.create();
            configuration.set("hbase.zookeeper.quorum", "192.168.1.101");
            configuration.set("hbase.zookeeper.property.clientPort", "2181");
    
            //集群配置↓
            //configuration.set("hbase.zookeeper.quorum", "101.236.39.141,101.236.46.114,101.236.46.113");
    //        configuration.set("hbase.master", "192.168.1.101:60000");
            Connection connection = ConnectionFactory.createConnection(configuration);
            return connection;
        }
    
    
        public static void scanDataStep1(String tableName) throws Exception {
    
            Table table= initHbase().getTable(TableName.valueOf(tableName));
            Scan scan = new Scan();
            ResultScanner scanner = table.getScanner(scan);
            for (Result result : scanner) {
                if (result.getValue(Bytes.toBytes("info"), Bytes.toBytes("f_pid")) == null) {
                    for (KeyValue kv : result.raw()) {
                        System.out.print(new String(kv.getRow()) + " ");
                        System.out.print(new String(kv.getFamily()) + ":");
                        System.out.print(new String(kv.getQualifier()) + " = ");
                        System.out.print(new String(kv.getValue()));
                        System.out.print(" timestamp = " + kv.getTimestamp() + "\n");
                    }
                }
            }
        }
    
        //创建表
        public static void createTable(String tableNmae, String[] cols) throws IOException {
    
            TableName tableName = TableName.valueOf(tableNmae);
            admin = initHbase().getAdmin();
            if (admin.tableExists(tableName)) {
                System.out.println("表已存在！");
            } else {
                HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
                for (String col : cols) {
                    HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(col);
                    hTableDescriptor.addFamily(hColumnDescriptor);
                }
                admin.createTable(hTableDescriptor);
            }
        }
    
    
        // 显示表结构
        public static void showTable(String tableNmae) throws Exception {
            admin = initHbase().getAdmin();
            HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(tableNmae));
            System.out.println("表结构--------------------------");
            byte[] name = tableDescriptor.getName();
            System.out.println("下面开始输出结果：");
            System.out.println("表名：" + new String(name));
            HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
            for (HColumnDescriptor d : columnFamilies)
            {
                System.out.println("列族名：" + d.getNameAsString());
            }
    
        }
    
        //插入数据
        public static void insertData(String tableName, User user) throws IOException {
            TableName tablename = TableName.valueOf(tableName);
            Put put = new Put(("user-" + user.getId()).getBytes());
            //参数：1.列族名  2.列名  3.值
            put.addColumn("information".getBytes(), "username".getBytes(), user.getUsername().getBytes()) ;
            put.addColumn("information".getBytes(), "age".getBytes(), user.getAge().getBytes()) ;
            put.addColumn("information".getBytes(), "gender".getBytes(), user.getGender().getBytes()) ;
            put.addColumn("contact".getBytes(), "phone".getBytes(), user.getPhone().getBytes());
            put.addColumn("contact".getBytes(), "email".getBytes(), user.getEmail().getBytes());
            //HTable table = new HTable(initHbase().getConfiguration(),tablename);已弃用
            Table table = initHbase().getTable(tablename);
            table.put(put);
        }
    
        //获取原始数据
        public static void getNoDealData(String tableName){
            try {
                Table table= initHbase().getTable(TableName.valueOf(tableName));
                Scan scan = new Scan();
                ResultScanner resutScanner = table.getScanner(scan);
                for(Result result: resutScanner){
                    System.out.println("scan:  " + result);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    
        // 得到所有数据,行,列簇,列名,值
        public static void getAllRows(String tableName) throws IOException{
    
            Table table = initHbase().getTable(TableName.valueOf(tableName));
            //得到用于扫描 region 的对象
            Scan scan = new Scan();
            //使用 HTable 得到 resultcanner 实现类的对象
            ResultScanner resultScanner = table.getScanner(scan);
            System.out.print("行键\t\t列族\t列\t值\n");
            for(Result result : resultScanner){
                Cell[] cells = result.rawCells();
                for(Cell cell : cells){
                    //得到 rowkey
                    System.out.print(Bytes.toString(CellUtil.cloneRow(cell)) +"\t");
                    //得到列族
                    System.out.print( Bytes.toString(CellUtil.cloneFamily(cell)) +"\t");
                    System.out.print( Bytes.toString(CellUtil.cloneQualifier(cell))+"\t");
                    System.out.println( Bytes.toString(CellUtil.cloneValue(cell)));
                }
                System.out.println("--------------------------------");
            }
        }
    
        //根据rowKey进行查询
        public static User getDataByRowKey(String tableName, String rowKey) throws IOException {
    
            Table table = initHbase().getTable(TableName.valueOf(tableName));
            Get get = new Get(rowKey.getBytes());
            User user = new User();
            user.setId(rowKey);
            //先判断是否有此条数据
            if(!get.isCheckExistenceOnly()){
                Result result = table.get(get);
                for (Cell cell : result.rawCells()){
                    String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength());
                    String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
                    if(colName.equals("username")){
                        user.setUsername(value);
                    }
                    if(colName.equals("age")){
                        user.setAge(value);
                    }
                    if (colName.equals("gender")){
                        user.setGender(value);
                    }
                    if (colName.equals("phone")){
                        user.setPhone(value);
                    }
                    if (colName.equals("email")){
                        user.setEmail(value);
                    }
                }
            }
            return user;
        }
    
        //查询指定单cell内容
        public static String getCellData(String tableName, String rowKey, String family, String col){
    
            try {
                Table table = initHbase().getTable(TableName.valueOf(tableName));
                String result = null;
                Get get = new Get(rowKey.getBytes());
                if(!get.isCheckExistenceOnly()){
                    get.addColumn(Bytes.toBytes(family),Bytes.toBytes(col));
                    Result res = table.get(get);
                    byte[] resByte = res.getValue(Bytes.toBytes(family), Bytes.toBytes(col));
                    return result = Bytes.toString(resByte);
                }else{
                    return result = "查询结果不存在";
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
            return "出现异常";
        }
    
        //查询指定表名中所有的数据
        public static List<User> getAllData(String tableName){
    
            Table table = null;
            List<User> list = new ArrayList<User>();
            try {
                table = initHbase().getTable(TableName.valueOf(tableName));
                ResultScanner results = table.getScanner(new Scan());
                User user = null;
                for (Result result : results){
                    String id = new String(result.getRow());
                    System.out.println("用户名:" + new String(result.getRow()));
                    user = new User();
                    for(Cell cell : result.rawCells()){
                        String row = Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
                        //String family =  Bytes.toString(cell.getFamilyArray(),cell.getFamilyOffset(),cell.getFamilyLength());
                        String colName = Bytes.toString(cell.getQualifierArray(),cell.getQualifierOffset(),cell.getQualifierLength());
                        String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
                        user.setId(row);
                        if(colName.equals("username")){
                            user.setUsername(value);
                        }
                        if(colName.equals("age")){
                            user.setAge(value);
                        }
                        if (colName.equals("gender")){
                            user.setGender(value);
                        }
                        if (colName.equals("phone")){
                            user.setPhone(value);
                        }
                        if (colName.equals("email")){
                            user.setEmail(value);
                        }
                    }
                    list.add(user);
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
            return list;
        }
    
        //删除指定cell数据
        public static void deleteByRowKey(String tableName, String rowKey) throws IOException {
    
            Table table = initHbase().getTable(TableName.valueOf(tableName));
            Delete delete = new Delete(Bytes.toBytes(rowKey));
            //删除指定列
            //delete.addColumns(Bytes.toBytes("contact"), Bytes.toBytes("email"));
            table.delete(delete);
        }
    
        //删除表
        public static void deleteTable(String tableName){
    
            try {
                TableName tablename = TableName.valueOf(tableName);
                admin = initHbase().getAdmin();
                admin.disableTable(tablename);
                admin.deleteTable(tablename);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    
    
    
    
    }

## HBase 过滤器 ##

基础API中的查询操作在面对大量数据的时候是非常苍白的，这里Hbase提供了高级的查询方法：Filter。Filter可以根据簇、列、版本等更多的条件来对数据进行过滤，基于Hbase本身提供的三维有序（主键有序、列有序、版本有序），这些Filter可以高效的完成查询过滤的任务。带有Filter条件的RPC查询请求会把Filter分发到各个RegionServer，是一个服务器端（Server-side）的过滤器，这样也可以降低网络传输的压力。

要完成一个过滤的操作，至少需要两个参数。一个是抽象的操作符，Hbase提供了枚举类型的变量来表示这些抽象的操作符：LESS/LESS\_OR\_EQUAL/EQUAL/NOT\_EUQAL等；另外一个就是具体的比较器（Comparator），代表具体的比较逻辑，如果可以提高字节级的比较、字符串级的比较等。有了这两个参数，我们就可以清晰的定义筛选的条件，过滤数据。

抽象操作符（比较运算符）

LESS <
    
    LESS_OR_EQUAL <=
    
    EQUAL =
    
    NOT_EQUAL <>
    
    GREATER_OR_EQUAL >=
    
    GREATER >
    
    NO_OP 排除所有

比较器（指定比较机制）

BinaryComparator 按字节索引顺序比较指定字节数组，采用 Bytes.compareTo(byte[])
    
    BinaryPrefixComparator 跟前面相同，只是比较左端的数据是否相同
    
    NullComparator 判断给定的是否为空
    
    BitComparator 按位比较
    
    RegexStringComparator 提供一个正则的比较器，仅支持 EQUAL 和非 EQUAL
    
    SubstringComparator 判断提供的子串是否出现在 value 中

## java编码 ##

输出结果

user-006/contact:email/1546667518090/Put/vlen=16/seqid=0
    user-006/contact:phone/1546667518090/Put/vlen=11/seqid=0
    user-006/information:age/1546667518090/Put/vlen=2/seqid=0
    user-006/information:gender/1546667518090/Put/vlen=3/seqid=0
    user-006/information:password/1546667518090/Put/vlen=6/seqid=0
    user-006/information:username/1546667518090/Put/vlen=7/seqid=0
    user-007/contact:email/1546667518117/Put/vlen=16/seqid=0
    user-007/contact:phone/1546667518117/Put/vlen=11/seqid=0
    user-007/information:age/1546667518117/Put/vlen=2/seqid=0
    user-007/information:gender/1546667518117/Put/vlen=6/seqid=0
    user-007/information:password/1546667518117/Put/vlen=6/seqid=0
    user-007/information:username/1546667518117/Put/vlen=7/seqid=0
    user-008/contact:email/1546667518144/Put/vlen=16/seqid=0
    user-008/contact:phone/1546667518144/Put/vlen=11/seqid=0
    user-008/information:age/1546667518144/Put/vlen=2/seqid=0
    user-008/information:gender/1546667518144/Put/vlen=3/seqid=0
    user-008/information:password/1546667518144/Put/vlen=6/seqid=0
    user-008/information:username/1546667518144/Put/vlen=7/seqid=0
    user-009/contact:email/1546667518173/Put/vlen=16/seqid=0
    user-009/contact:phone/1546667518173/Put/vlen=11/seqid=0
    user-009/information:age/1546667518173/Put/vlen=2/seqid=0
    user-009/information:gender/1546667518173/Put/vlen=6/seqid=0
    user-009/information:password/1546667518173/Put/vlen=6/seqid=0
    user-009/information:username/1546667518173/Put/vlen=7/seqid=0
    列簇过滤器 FamilyFilter :过滤info信息----------------------------------------------------------------------
    user-000/contact:email/1546667517904/Put/vlen=16/seqid=0
    user-000/contact:phone/1546667517904/Put/vlen=11/seqid=0
    user-001/contact:email/1546667517960/Put/vlen=16/seqid=0
    user-001/contact:phone/1546667517960/Put/vlen=11/seqid=0
    user-002/contact:email/1546667517996/Put/vlen=16/seqid=0
    user-002/contact:phone/1546667517996/Put/vlen=11/seqid=0
    user-003/contact:email/1546667518018/Put/vlen=16/seqid=0
    user-003/contact:phone/1546667518018/Put/vlen=11/seqid=0
    user-004/contact:email/1546667518041/Put/vlen=16/seqid=0
    user-004/contact:phone/1546667518041/Put/vlen=11/seqid=0
    user-005/contact:email/1546667518067/Put/vlen=16/seqid=0
    user-005/contact:phone/1546667518067/Put/vlen=11/seqid=0
    user-006/contact:email/1546667518090/Put/vlen=16/seqid=0
    user-006/contact:phone/1546667518090/Put/vlen=11/seqid=0
    user-007/contact:email/1546667518117/Put/vlen=16/seqid=0
    user-007/contact:phone/1546667518117/Put/vlen=11/seqid=0
    user-008/contact:email/1546667518144/Put/vlen=16/seqid=0
    user-008/contact:phone/1546667518144/Put/vlen=11/seqid=0
    user-009/contact:email/1546667518173/Put/vlen=16/seqid=0
    user-009/contact:phone/1546667518173/Put/vlen=11/seqid=0
    值过滤器 ValueFilter :过滤性别为女的数据----------------------------------------------------------------------
    user-001/information:gender/1546667517960/Put/vlen=6/seqid=0
    user-003/information:gender/1546667518018/Put/vlen=6/seqid=0
    user-005/information:gender/1546667518067/Put/vlen=6/seqid=0
    user-007/information:gender/1546667518117/Put/vlen=6/seqid=0
    user-009/information:gender/1546667518173/Put/vlen=6/seqid=0
    时间戳过滤器 TimestampsFilter ValueFilter :过滤时间戳为1546665288963的数据----------------------------------------------------------------------
    单列值过滤器 SingleColumnValueFilter :过滤名字为hbase_4的数据----------------------------------------------------------------------
    user-004	contact	email	base_4@hbase.com	1546667518041
    user-004	contact	phone	13355550024	1546667518041
    user-004	information	age	20	1546667518041
    user-004	information	gender	man	1546667518041
    user-004	information	password	123456	1546667518041
    user-004	information	username	hbase_4	1546667518041
    前缀过滤器 PrefixFilter----只对行键有用----------------------------------------------------------------------
    user-000	contact	email	base_0@hbase.com	1546667517904
    user-000	contact	phone	13355550020	1546667517904
    user-000	information	age	20	1546667517904
    user-000	information	gender	man	1546667517904
    user-000	information	password	123456	1546667517904
    user-000	information	username	hbase_0	1546667517904
    user-001	contact	email	base_1@bmaoo.com	1546667517960
    user-001	contact	phone	13355550021	1546667517960
    user-001	information	age	21	1546667517960
    user-001	information	gender	female	1546667517960
    user-001	information	password	123456	1546667517960
    user-001	information	username	hbase_1	1546667517960
    user-002	contact	email	base_2@hbase.com	1546667517996
    user-002	contact	phone	13355550022	1546667517996
    user-002	information	age	20	1546667517996
    user-002	information	gender	man	1546667517996
    user-002	information	password	123456	1546667517996
    user-002	information	username	hbase_2	1546667517996
    user-003	contact	email	base_3@bmaoo.com	1546667518018
    user-003	contact	phone	13355550023	1546667518018
    user-003	information	age	21	1546667518018
    user-003	information	gender	female	1546667518018
    user-003	information	password	123456	1546667518018
    user-003	information	username	hbase_3	1546667518018
    user-004	contact	email	base_4@hbase.com	1546667518041
    user-004	contact	phone	13355550024	1546667518041
    user-004	information	age	20	1546667518041
    user-004	information	gender	man	1546667518041
    user-004	information	password	123456	1546667518041
    user-004	information	username	hbase_4	1546667518041
    user-005	contact	email	base_5@bmaoo.com	1546667518067
    user-005	contact	phone	13355550025	1546667518067
    user-005	information	age	21	1546667518067
    user-005	information	gender	female	1546667518067
    user-005	information	password	123456	1546667518067
    user-005	information	username	hbase_5	1546667518067
    user-006	contact	email	base_6@hbase.com	1546667518090
    user-006	contact	phone	13355550026	1546667518090
    user-006	information	age	20	1546667518090
    user-006	information	gender	man	1546667518090
    user-006	information	password	123456	1546667518090
    user-006	information	username	hbase_6	1546667518090
    user-007	contact	email	base_7@bmaoo.com	1546667518117
    user-007	contact	phone	13355550027	1546667518117
    user-007	information	age	21	1546667518117
    user-007	information	gender	female	1546667518117
    user-007	information	password	123456	1546667518117
    user-007	information	username	hbase_7	1546667518117
    user-008	contact	email	base_8@hbase.com	1546667518144
    user-008	contact	phone	13355550028	1546667518144
    user-008	information	age	20	1546667518144
    user-008	information	gender	man	1546667518144
    user-008	information	password	123456	1546667518144
    user-008	information	username	hbase_8	1546667518144
    user-009	contact	email	base_9@bmaoo.com	1546667518173
    user-009	contact	phone	13355550029	1546667518173
    user-009	information	age	21	1546667518173
    user-009	information	gender	female	1546667518173
    user-009	information	password	123456	1546667518173
    user-009	information	username	hbase_9	1546667518173
    列前缀过滤器 ColumnPrefixFilter匹配password,phone两个----------------------------------------------------------------------
    user-000	contact	phone	13355550020	1546667517904
    user-000	information	password	123456	1546667517904
    user-001	contact	phone	13355550021	1546667517960
    user-001	information	password	123456	1546667517960
    user-002	contact	phone	13355550022	1546667517996
    user-002	information	password	123456	1546667517996
    user-003	contact	phone	13355550023	1546667518018
    user-003	information	password	123456	1546667518018
    user-004	contact	phone	13355550024	1546667518041
    user-004	information	password	123456	1546667518041
    user-005	contact	phone	13355550025	1546667518067
    user-005	information	password	123456	1546667518067
    user-006	contact	phone	13355550026	1546667518090
    user-006	information	password	123456	1546667518090
    user-007	contact	phone	13355550027	1546667518117
    user-007	information	password	123456	1546667518117
    user-008	contact	phone	13355550028	1546667518144
    user-008	information	password	123456	1546667518144
    user-009	contact	phone	13355550029	1546667518173
    user-009	information	password	123456	1546667518173

更多hadoop进阶源码请参看我的github地址

springboot-habse操作整合  
[https://gitee.com/susonglin/springboot\_hbase\_kafka][https_gitee.com_susonglin_springboot_hbase_kafka]

[https_gitee.com_susonglin_springboot_hbase_kafka]: https://gitee.com/susonglin/springboot_hbase_kafka