flume

multiple flows

A single Flume agent can contain several independent flows. You can list multiple sources, sinks and channels in a config. These components can be linked to form multiple flows:

客户端

agent1.channels = mc1 mc2
agent1.sources = ngrinder ngrinder2
agent1.sinks = avro-sink avro-sink2


#mc1
agent1.channels.mc1.type = file
agent1.channels.mc1.checkpointDir = /usr/local/soft/flume/checkpoint1
agent1.channels.mc1.dataDirs = /usr/local/soft/flume/data1

agent1.sources.ngrinder.type = spooldir
agent1.sources.ngrinder.channels = mc1
agent1.sources.ngrinder.spoolDir = /usr/local/soft/flume/sp1
agent1.sources.ngrinder.fileHeader = true
agent1.sources.ngrinder.basenameHeader = true

agent1.sinks.avro-sink.channel = mc1
agent1.sinks.avro-sink.type = avro
agent1.sinks.avro-sink.port = 4545
agent1.sinks.avro-sink.hostname = 192.168.137.106


#mc2
agent1.channels.mc2.type = file
agent1.channels.mc2.checkpointDir = /usr/local/soft/flume/checkpoint2
agent1.channels.mc2.dataDirs = /usr/local/soft/flume/data2

agent1.sources.ngrinder2.type = spooldir
agent1.sources.ngrinder2.channels = mc2
agent1.sources.ngrinder2.spoolDir = /usr/local/soft/flume/sp2
agent1.sources.ngrinder2.fileHeader = true
agent1.sources.ngrinder2.basenameHeader = true

agent1.sinks.avro-sink2.channel = mc2
agent1.sinks.avro-sink2.type = avro
agent1.sinks.avro-sink2.port = 4546
agent1.sinks.avro-sink2.hostname = 192.168.137.106

　　

服务端

collector1.sources = AvroIn AvroIn2
collector1.channels = mc1 mc2
collector1.sinks = LocalOut LocalOut2

#mc1
collector1.sources.AvroIn.type = avro
collector1.sources.AvroIn.bind = 0.0.0.0
collector1.sources.AvroIn.port = 4545
collector1.sources.AvroIn.channels = mc1


collector1.channels.mc1.type = memory
collector1.channels.mc1.checkpointDir = /usr/local/soft/flume/checkpoint1
collector1.channels.mc1.dataDirs = /usr/local/soft/flume/data1
collector1.channels.mc1.capacity = 100


collector1.sinks.LocalOut.type = file_roll
collector1.sinks.LocalOut.sink.directory = /usr/local/soft/flume/collector1
collector1.sinks.LocalOut.channel = mc1

#mc2
collector1.sources.AvroIn2.type = avro
collector1.sources.AvroIn2.bind = 0.0.0.0
collector1.sources.AvroIn2.port = 4546
collector1.sources.AvroIn2.channels = mc2


collector1.channels.mc2.type = memory
collector1.channels.mc2.checkpointDir = /usr/local/soft/flume/checkpoint2
collector1.channels.mc2.dataDirs = /usr/local/soft/flume/data2
collector1.channels.mc2.capacity = 100


collector1.sinks.LocalOut2.type = file_roll
collector1.sinks.LocalOut2.sink.directory = /usr/local/soft/flume/collector2
collector1.sinks.LocalOut2.channel = mc2