由于使用的是HA,两个namenode会不定时休眠其中一个,为了从业务主机上采集的数据不丢失,配置了两个sink,但是无法自动切换,配置文件如下:
a00.sources = r00
a00.sinks = k00_1 k00_2
a00.channels = c00
# Describe/configure the source
a00.sources.r00.type = exec
a00.sources.r00.command = tail -F /home/flume/others/test/20190419_test/apvtest.log
a00.sources.r00.channels = c00
#set gruop
a00.sinkgroups = g00
a00.sinkgroups.g00.sinks = k00_1 k00_2
#set failove
a00.sinkgroups.g00.processor.type = failove
a00.sinkgroups.g00.processor.priority.k00_1 = 10
a00.sinkgroups.g00.processor.priority.k00_2 = 5
#最长等待10秒转移故障20190416
a00.sinkgroups.g00.processor.maxpenalty = 10000
# set sink1
a00.sinks.k00_1.type = hdfs
a00.sinks.k00_1.hdfs.path = hdfs://***.***.***.157:9000/flume/log-test/%Y%m%d
a00.sinks.k00_1.hdfs.filePrefix = apv3
a00.sinks.k00_1.hdfs.fileSuffix = .log
a00.sinks.k00_1.hdfs.batchSize= 500
a00.sinks.k00_1.hdfs.fileType = DataStream
a00.sinks.k00_1.hdfs.writeFormat =Text
a00.sinks.k00_1.hdfs.rollSize = 0
a00.sinks.k00_1.hdfs.rollCount = 0
a00.sinks.k00_1.hdfs.rollInterval = 100
a00.sinks.k00_1.hdfs.round = true
a00.sinks.k00_1.hdfs.roundValue = 24
a00.sinks.k00_1.hdfs.roundUnit = houre
a00.sinks.k00_1.hdfs.useLocalTimeStamp = true
# set sink2
a00.sinks.k00_2.type = hdfs
a00.sinks.k00_2.hdfs.path = hdfs://***.***.***.165:9000/flume/log-test/%Y%m%d
a00.sinks.k00_2.hdfs.filePrefix = apv3
a00.sinks.k00_2.hdfs.fileSuffix = .log
a00.sinks.k00_2.hdfs.batchSize= 500
a00.sinks.k00_2.hdfs.fileType = DataStream
a00.sinks.k00_2.hdfs.writeFormat =Text
a00.sinks.k00_2.hdfs.rollSize = 0
a00.sinks.k00_2.hdfs.rollCount = 0
a00.sinks.k00_2.hdfs.rollInterval = 100
a00.sinks.k00_2.hdfs.round = true
a00.sinks.k00_2.hdfs.roundValue = 24
a00.sinks.k00_2.hdfs.roundUnit = houre
a00.sinks.k00_2.hdfs.useLocalTimeStamp = true
# Use a channel which buffers events in memory
a00.channels.c00.type = memory
a00.channels.c00.capacity = 50000
a00.channels.c00.transactionCapacity = 6000
# Bind the source and sink to the channel
a00.sinks.k00_1.channel = c00
a00.sinks.k00_2.channel = c00
。但是当157的节点休眠时,还是不能采集数据,日志中一直报错,节点休眠.......请问如何解决?
相似问题