使用datax将hive中的数据导入到clickhouse

我不是女神ヾ 2023-01-19 04:54 803阅读 0赞

{ 
      "core": {        
        "transport": {           
          "channel": {                
            "speed": {                    
              "byte": 10485760      
                    }
                }
            }
        },      
        "job": {         
            "setting": {            
                "speed": {                
                    "byte":10485760           
                },         
                "errorLimit": {               
                    "record": 0,             
                    "percentage": 0.02         
                }      
            },      
            "content": [
                { 
                    "reader": { 
                        "name": "hdfsreader",
                        "parameter": { 
                "hadoopConfig": { 
                                "dfs.nameservices": "tqHadoopCluster",
                                "dfs.ha.namenodes.tqHadoopCluster": "nn1,nn2",
                                "dfs.namenode.rpc-address.tqHadoopCluster.nn1": "xxx.xxx.xxx.xxx:8020",
                                "dfs.namenode.rpc-address.tqHadoopCluster.nn2": "xxx.xxx.xxx.xxx:8020",
                                "dfs.client.failover.proxy.provider.tqHadoopCluster": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
                            },
                "defaultFS": "hdfs://xxxxx",
                            "path": "/xxxxx_3/hiveWarehouse/population_tag_test",
                            "column": [
                  { 
                                    "index": 0,
                                    "type": "string"
                                },
                  { 
                                    "index": 1,
                                    "type": "string"
                                },
                  { 
                                    "index": 2,
                                    "type": "string"
                                },
                  { 
                                    "index": 3,
                                    "type": "string"
                                },
                  { 
                                    "index": 4,
                                    "type": "string"
                                },
                  { 
                                    "index": 5,
                                    "type": "string"
                                },
                  { 
                                    "index": 6,
                                    "type": "string"
                                }
                ],
                            "fileType": "text",
                            "encoding": "UTF-8",
                            "fieldDelimiter": ",",
                "nullFormat":"\\N"
                        }
                    },
                   "writer": { 
              "name": "clickhousewriter",
              "parameter": { 
                "username": "default",
                "password":"tianquekeji",
                "column":["one_idcardnumber","one_name","one_sex","one_father","one_mother","one_landlord","one_disable_guardian"],
                "connection": [
                  { 
                    "jdbcUrl": "jdbc:clickhouse://xxxxxx:8123/test",
                    "table":["population_tag"]
                  }
                ]
              }
            }
                }
            ]
        }
    }

python $DATAX\_HOME/bin/datax.sh xxx.json