hbase数据转移: 导入导出

我不是女神ヾ 2022-03-31 07:47 967阅读 0赞

### 文章目录 ###

*  hbase数据“摧毁重建”
 *   *  a, 两个集群版本相同，但网络不通：数据在主机间scp
     *  b, 两个集群版本相同，且网络能通：hadoop distcp 数据
 *  0，hbase java api导入数据： scan-- >put
 *  1，hbase数据import , export
 *  2，hbase集群版本一致：copyTable
 *  3，导入csv文件到hbase: bulckload

# hbase数据“摧毁重建” #

以“摧毁重建”的方式，完成hbase的新数据导入

## a, 两个集群版本相同，但网络不通：数据在主机间scp ##

#======src 数据源端======
    #1,下载hbase某个表的数据
    hdfs dfs -get /hbase/data/default/t1
    #2, 传输文件到 远程主机
    scp -r t1 cdh1:/
    
    #======dest 数据接收端======
    #1, 上传hbase数据文件
    hdfs dfs -put t1 /hbase/data/default
    #2,参考博客，执行hbase数据“摧毁重建”
    https://blog.csdn.net/eyeofeagle/article/details/88343805

## b, 两个集群版本相同，且网络能通：hadoop distcp 数据 ##

#1, 上传hbase数据到新集群
     export HADOOP_CLIENT_OPTS="-Xms2048m -Xmx4048m"
     hadoop distcp --update hdfs://cdh1:8020/hbase/data/default  hdfs://cdh2:8020/tmp/hbase_data_default
    #2,参考博客，执行hbase数据“摧毁重建”
    https://blog.csdn.net/eyeofeagle/article/details/88343805

# 0，hbase java api导入数据： scan-- >put #

使用hbase java api先读取hbase1的数据，然后写入hbase2的表：[https://blog.csdn.net/eyeofeagle/article/details/83614913][https_blog.csdn.net_eyeofeagle_article_details_83614913]

# 1，hbase数据import , export #

##方式1 （Export ）： 导出数据, 导入数据时需先建好hbase表
    hbase org.apache.hadoop.hbase.mapreduce.Export  表名   mapreduce文件路径  
    hbase org.apache.hadoop.hbase.mapreduce.Import  表名   mapreduce文件路径
    
    ##方式2 （Driver）： 导出数据, 导入数据时需先建好hbase表
    #hbase org.apache.hadoop.hbase.mapreduce.Driver import    表名    mapreduce文件路径  
    #hbase org.apache.hadoop.hbase.mapreduce.Driver export    表名    mapreduce文件路径  
    
    ##具体操作
    #===================src 数据源端 ===============
    hbase(main):003:0> desc 'test'
    Table test is ENABLED                                                                                                                         
    test                                                                                                                                          
    COLUMN FAMILIES DESCRIPTION                                                                                                                   
    {NAME => 'f1', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL 
    => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}               
    {NAME => 'f2', BLOOMFILTER => 'ROW', VERSIONS => '1', IN_MEMORY => 'false', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL 
    => 'FOREVER', COMPRESSION => 'NONE', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}               
    2 row(s) in 0.0600 seconds
    
    hbase(main):002:0> scan 'test'
    ROW                                  COLUMN+CELL                                                                                              
     r1                                  column=f1:age, timestamp=1566455175438, value=23                                                         
     r1                                  column=f1:name, timestamp=1566455169109, value=a                                                         
     r2                                  column=f1:age, timestamp=1566455181784, value=33                                                         
     r2                                  column=f1:job, timestamp=1566455199238, value=java                                                       
     r2                                  column=f1:name, timestamp=1566455191650, value=lisi                                                      
     r2                                  column=f2:addr, timestamp=1566455218133, value=ch-guangdong                                              
     r3                                  column=f1:name, timestamp=1569221612403, value=a                                                         
    3 row(s) in 0.1600 seconds
    
    [root@eadage ~]# hbase org.apache.hadoop.hbase.mapreduce.Export   test   file:///tmp/test.hbase
    ....
    19/09/23 16:20:27 INFO mapreduce.Job: Job job_1568012065562_0281 running in uber mode : false
    19/09/23 16:20:27 INFO mapreduce.Job:  map 0% reduce 0%
    19/09/23 16:20:34 INFO mapreduce.Job:  map 100% reduce 0%
    19/09/23 16:20:34 INFO mapreduce.Job: Job job_1568012065562_0281 completed successfully
    19/09/23 16:20:34 INFO mapreduce.Job: Counters: 30
    	Map-Reduce Framework
    		Map input records=3
    		Map output records=3
    
    
    [root@test1 ~]# ls /tmp/test.hbase/
    part-m-00000  _SUCCESS
    
    [root@test1 ~]# scp -r  /tmp/test.hbase cdh1:/tmp/
    Warning: Permanently added 'cdh1,192.168.58.151' (RSA) to the list of known hosts.
    ._SUCCESS.crc                                                                                               100%    8     0.0KB/s   00:00    
    .part-m-00000.crc                                                                                           100%   12     0.0KB/s   00:00    
    _SUCCESS                                                                                                    100%    0     0.0KB/s   00:00    
    part-m-00000                                                                                                100%  384     0.4KB/s   00:00    
    
    
    #=================dest  数据目标端 ===============
    [root@cdh1 /]# chown -R hdfs:hdfs /tmp/test.hbase/
    [root@cdh1 /]# ls /tmp/test.hbase/part-m-00000
    /tmp/test.hbase/part-m-00000
    
    hbase(main):008:0> create 'test','f1','f2'
    0 row(s) in 2.3000 seconds
    => Hbase::Table - test
    
    [hdfs@cdh1 root]$ hbase org.apache.hadoop.hbase.mapreduce.Import  test file:///tmp/test.hbase/
    ....
    19/09/23 16:31:30 INFO mapreduce.Job:  map 100% reduce 0%
    19/09/23 16:31:30 INFO mapreduce.Job: Job job_1569224609141_0003 completed successfully
    19/09/23 16:31:30 INFO mapreduce.Job: Counters: 30
    	Map-Reduce Framework
    		Map input records=3
    		Map output records=3
    	
    
    hbase(main):002:0> scan 'test'
    ROW                                  COLUMN+CELL                                                                                              
     r1                                  column=f1:age, timestamp=1566455175438, value=23                                                         
     r1                                  column=f1:name, timestamp=1566455169109, value=a                                                         
     r2                                  column=f1:age, timestamp=1566455181784, value=33                                                         
     r2                                  column=f1:job, timestamp=1566455199238, value=java                                                       
     r2                                  column=f1:name, timestamp=1566455191650, value=lisi                                                      
     r2                                  column=f2:addr, timestamp=1566455218133, value=ch-guangdong                                              
     r3                                  column=f1:name, timestamp=1569221612403, value=a                                                         
    3 row(s) in 0.2320 seconds

# 2，hbase集群版本一致：copyTable #

#相同集群:  新的表要提前创建好，如"ns1:xx", xx
     hbase org.apache.hadoop.hbase.mapreduce.CopyTable --new.name=tableCopy  srcTable
        #不同集群， 新的表要提前创建好
     hbase org.apache.hadoop.hbase.mapreduce.CopyTable --peer.adr=dstClusterZK:2181:/hbase srcTable
    
    #hbase不同集群的数据拷贝  示例
    hbase org.apache.hadoop.hbase.mapreduce.CopyTable --peer.adr=172.17.0.3:2181:/hbase  --new.name=class3 --families=f:f3  --starttime=1 --endtime=1550212874479     class
    
    日志如下：
    2019-02-15 06:28:46,726 INFO  [main] zookeeper.ReadOnlyZKClient: Connect 0x7e928e2f to localhost:2181 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms
    2019-02-15 06:28:46,730 INFO  [ReadOnlyZKClient-localhost:2181@0x7e928e2f] zookeeper.ZooKeeper: Initiating client connection, connectString=localhost:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$13/240114419@3fc4eee8
    2019-02-15 06:28:46,731 INFO  [ReadOnlyZKClient-localhost:2181@0x7e928e2f-SendThread(localhost:2181)] zookeeper.ClientCnxn: Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error)
    2019-02-15 06:28:46,732 INFO  [ReadOnlyZKClient-localhost:2181@0x7e928e2f-SendThread(localhost:2181)] zookeeper.ClientCnxn: Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session
    2019-02-15 06:28:46,747 INFO  [ReadOnlyZKClient-localhost:2181@0x7e928e2f-SendThread(localhost:2181)] zookeeper.ClientCnxn: Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x168eee4c1d60018, negotiated timeout = 90000
    2019-02-15 06:28:46,751 INFO  [main] mapreduce.RegionSizeCalculator: Calculating region sizes for table "class".
    2019-02-15 06:28:46,928 INFO  [main] zookeeper.ReadOnlyZKClient: Close zookeeper connection 0x7e928e2f to localhost:2181
    2019-02-15 06:28:46,938 INFO  [ReadOnlyZKClient-localhost:2181@0x7e928e2f] zookeeper.ZooKeeper: Session: 0x168eee4c1d60018 closed
    2019-02-15 06:28:46,939 INFO  [ReadOnlyZKClient-localhost:2181@0x7e928e2f-EventThread] zookeeper.ClientCnxn: EventThread shut down for session: 0x168eee4c1d60018
    2019-02-15 06:28:46,982 INFO  [main] mapreduce.JobSubmitter: number of splits:1
    2019-02-15 06:28:47,176 INFO  [main] mapreduce.JobSubmitter: Submitting tokens for job: job_local1064070611_0001

# 3，导入csv文件到hbase: bulckload #

##建表
    echo  "create 'wordcount', {NAME => 'f'},   {SPLITS => ['g', 'm', 'r', 'w']} "  |hbaes shell
    
    ##etl数据转换
    curl  https://people.apache.org/~jdcryans/word_count.csv
    hdfs   dfs   -put word_count.csv
    hbase  org.apache.hadoop.hbase.mapreduce.ImportTsv \
    -Dimporttsv.separator=, \
    -Dimporttsv.bulk.output=output \
    -Dimporttsv.columns=HBASE_ROW_KEY,f:count    wordcount    word_count.csv
    
    ## 加载hfile数据到hbase表中:  数据格式 output/f/c656d893bd704260a613be62bddb4d5f
    hbase  org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles  output  hbase表名

[https_blog.csdn.net_eyeofeagle_article_details_83614913]: https://blog.csdn.net/eyeofeagle/article/details/83614913