flume采集日志文件到hdfs

叁歲伎倆 2022-02-21 07:47 407阅读 0赞

### flume采集日志文件到hdfs ###

*  配置文件
 *  启动命令
 *  运行结果

> flume可以从正在输出的日志文件中采集数据存到hdfs中

# 配置文件 #

agent1.sources = source1
    agent1.sinks = sink1
    agent1.channels = channel1
    
    # Describe/configure tail -F source1
    agent1.sources.source1.type = exec
    agent1.sources.source1.command = tail -F /root/app/hadoop-2.6.4/logs/hadoop-root-namenode-zk1.log
    agent1.sources.source1.channels = channel1
    
    #configure host for source
    agent1.sources.source1.interceptors = i1
    agent1.sources.source1.interceptors.i1.type = host
    agent1.sources.source1.interceptors.i1.hostHeader = hostname
    
    # Describe sink1
    agent1.sinks.sink1.type = hdfs
    #a1.sinks.k1.channel = c1
    agent1.sinks.sink1.hdfs.path =hdfs://zk1:9000/weblog/flume-collection/%y-%m-%d/%H-%M
    agent1.sinks.sink1.hdfs.filePrefix = access_log
    agent1.sinks.sink1.hdfs.maxOpenFiles = 5000
    agent1.sinks.sink1.hdfs.batchSize= 100
    agent1.sinks.sink1.hdfs.fileType = DataStream
    agent1.sinks.sink1.hdfs.writeFormat =Text
    agent1.sinks.sink1.hdfs.rollSize = 102400
    agent1.sinks.sink1.hdfs.rollCount = 1000000
    agent1.sinks.sink1.hdfs.rollInterval = 60
    agent1.sinks.sink1.hdfs.round = true
    agent1.sinks.sink1.hdfs.roundValue = 10
    agent1.sinks.sink1.hdfs.roundUnit = minute
    agent1.sinks.sink1.hdfs.useLocalTimeStamp = true
    
    # Use a channel which buffers events in memory
    agent1.channels.channel1.type = memory
    agent1.channels.channel1.keep-alive = 120
    agent1.channels.channel1.capacity = 500000
    agent1.channels.channel1.transactionCapacity = 600
    
    # Bind the source and sink to the channel
    agent1.sources.source1.channels = channel1
    agent1.sinks.sink1.channel = channel1

# 启动命令 #

bin/flume-ng agent -c conf -f conf/log2hdfs.conf -n agent1

# 运行结果 #

hdfs中的文件一直在增加

[root@zk2 hadoop]# hdfs dfs -ls /weblog/flume-collection/19-04-11/00-00
    Found 4 items
    -rw-r--r--   2 root supergroup       2287 2019-04-11 00:04 /weblog/flume-collection/19-04-11/00-00/access_log.1554912190981
    -rw-r--r--   2 root supergroup       4373 2019-04-11 00:05 /weblog/flume-collection/19-04-11/00-00/access_log.1554912253151
    -rw-r--r--   2 root supergroup       2552 2019-04-11 00:06 /weblog/flume-collection/19-04-11/00-00/access_log.1554912313318
    -rw-r--r--   2 root supergroup          0 2019-04-11 00:06 /weblog/flume-collection/19-04-11/00-00/access_log.1554912373510.tmp
    [root@zk2 hadoop]# hdfs dfs -ls /weblog/flume-collection/19-04-11/00-00
    Found 5 items
    -rw-r--r--   2 root supergroup       2287 2019-04-11 00:04 /weblog/flume-collection/19-04-11/00-00/access_log.1554912190981
    -rw-r--r--   2 root supergroup       4373 2019-04-11 00:05 /weblog/flume-collection/19-04-11/00-00/access_log.1554912253151
    -rw-r--r--   2 root supergroup       2552 2019-04-11 00:06 /weblog/flume-collection/19-04-11/00-00/access_log.1554912313318
    -rw-r--r--   2 root supergroup       2552 2019-04-11 00:07 /weblog/flume-collection/19-04-11/00-00/access_log.1554912373510
    -rw-r--r--   2 root supergroup          0 2019-04-11 00:07 /weblog/flume-collection/19-04-11/00-00/access_log.1554912433704.tmp