Flume-ng生产环境实践（三）实现文件sink，按照固定格式目录输出

应用场景：需要实时收集多台服务器的nginx日志到一台机器。收集完成结果存放需要按天生成文件夹，按每5分钟生成文件，比如2012年12月29日12点26分的日志，需要放到/data/log/20121229/log-1225-对应的文件中。自己实现了类似flume-og和flume-ng的hdfs-sink的文件sink。

使用的时候配置如下：

agent.sources = source agent.channels = channel agent.sinks = sink agent.sources.source.type = avro agent.sources.source.bind = 192.168.0.100 agent.sources.source.port = 44444 agent.sources.source.channels = channel agent.sinks.sink.type = org.apache.flume.sink.FileSink agent.sinks.sink.file.path = /data/log/%{dayStr} agent.sinks.sink.file.filePrefix = log-%{hourStr}%{minStr}- agent.sinks.sink.file.txnEventMax = 10000 agent.sinks.sink.file.maxOpenFiles = 5 agent.sinks.sink.channel = channel agent.channels.channel.type = memory agent.channels.channel.capacity = 100000 agent.channels.channel.transactionCapacity = 100000 agent.channels.channel.keep-alive = 60

依赖的jar如下：

jakarta-oro-2.0.1.jar

flume-ng-core-1.3.0-SNAPSHOT.jar

flume-ng-sdk-1.3.0-SNAPSHOT.jar

flume-ng-configuration-1.3.0-SNAPSHOT.jar

slf4j-log4j12-1.6.1.jar

slf4j-api-1.6.1.jar

guava-10.0.1.jar

代码如下：

FileSink.java

package org.apache.flume.sink; import java.io.IOException; import java.util.Calendar; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import org.apache.flume.Channel; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.EventDeliveryException; import org.apache.flume.Transaction; import org.apache.flume.conf.Configurable; import org.apache.flume.formatter.output.BucketPath; import org.apache.flume.instrumentation.SinkCounter; import org.apache.flume.serialization.EventSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; public class FileSink extends AbstractSink implements Configurable { private static final Logger logger = LoggerFactory .getLogger(FileSink. class ); private String path ; private static final String defaultFileName = "FlumeData" ; private static final int defaultMaxOpenFiles = 50; /** * Default length of time we wait for blocking BucketWriter calls before * timing out the operation. Intended to prevent server hangs. */ private long txnEventMax ; private FileWriterLinkedHashMap sfWriters ; private String serializerType ; private Context serializerContext ; private boolean needRounding = false ; private int roundUnit = Calendar. SECOND ; private int roundValue = 1; private SinkCounter sinkCounter ; private int maxOpenFiles ; private ScheduledExecutorService timedRollerPool ; private long rollInterval ; @Override public void configure(Context context) { String directory = Preconditions.checkNotNull( context.getString( "file.path" ), "file.path is required" ); String fileName = context.getString( "file.filePrefix" , defaultFileName ); this . path = directory + "/" + fileName; maxOpenFiles = context.getInteger( "file.maxOpenFiles" , defaultMaxOpenFiles ); serializerType = context.getString( "sink.serializer" , "TEXT" ); serializerContext = new Context( context.getSubProperties(EventSerializer. CTX_PREFIX )); txnEventMax = context.getLong( "file.txnEventMax" , 1l); if ( sinkCounter == null ) { sinkCounter = new SinkCounter(getName()); } rollInterval = context.getLong( "file.rollInterval" , 30l); String rollerName = "hdfs-" + getName() + "-roll-timer-%d" ; timedRollerPool = Executors.newScheduledThreadPool( maxOpenFiles , new ThreadFactoryBuilder().setNameFormat(rollerName).build()); } @Override public Status process() throws EventDeliveryException { Channel channel = getChannel(); Transaction transaction = channel.getTransaction(); List<BucketFileWriter> writers = Lists.newArrayList(); transaction.begin(); try { Event event = null ; int txnEventCount = 0; for (txnEventCount = 0; txnEventCount < txnEventMax ; txnEventCount++) { event = channel.take(); if (event == null ) { break ; } // reconstruct the path name by substituting place holders String realPath = BucketPath .escapeString( path , event.getHeaders(), needRounding , roundUnit , roundValue ); BucketFileWriter bucketFileWriter = sfWriters .get(realPath); // we haven't seen this file yet, so open it and cache the // handle if (bucketFileWriter == null ) { bucketFileWriter = new BucketFileWriter(); bucketFileWriter.open(realPath, serializerType , serializerContext , rollInterval , timedRollerPool , sfWriters ); sfWriters .put(realPath, bucketFileWriter); } // track the buckets getting written in this transaction if (!writers.contains(bucketFileWriter)) { writers.add(bucketFileWriter); } // Write the data to File bucketFileWriter.append(event); } if (txnEventCount == 0) { sinkCounter .incrementBatchEmptyCount(); } else if (txnEventCount == txnEventMax ) { sinkCounter .incrementBatchCompleteCount(); } else { sinkCounter .incrementBatchUnderflowCount(); } // flush all pending buckets before committing the transaction for (BucketFileWriter bucketFileWriter : writers) { if (!bucketFileWriter.isBatchComplete()) { flush(bucketFileWriter); } } transaction.commit(); if (txnEventCount > 0) { sinkCounter .addToEventDrainSuccessCount(txnEventCount); } if (event == null ) { return Status. BACKOFF ; } return Status. READY ; } catch (IOException eIO) { transaction.rollback(); logger .warn( "File IO error" , eIO); return Status. BACKOFF ; } catch (Throwable th) { transaction.rollback(); logger .error( "process failed" , th); if (th instanceof Error) { throw (Error) th; } else { throw new EventDeliveryException(th); } } finally { transaction.close(); } } private void flush(BucketFileWriter bucketFileWriter) throws IOException { bucketFileWriter.flush(); } @Override public synchronized void start() { super .start(); this . sfWriters = new FileWriterLinkedHashMap( maxOpenFiles ); sinkCounter .start(); } }

BucketFileWriter.java

package org.apache.flume.sink; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.concurrent.Callable; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.serialization.EventSerializer; import org.apache.flume.serialization.EventSerializerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class BucketFileWriter { private static final Logger logger = LoggerFactory .getLogger(BucketFileWriter.class); private static final String IN_USE_EXT = ".tmp"; /** * This lock ensures that only one thread can open a file at a time. */ private final AtomicLong fileExtensionCounter; private OutputStream outputStream; private EventSerializer serializer; private String filePath; /** * Close the file handle and rename the temp file to the permanent filename. * Safe to call multiple times. Logs HDFSWriter.close() exceptions. * * @throws IOException * On failure to rename if temp file exists. */ public BucketFileWriter() { fileExtensionCounter = new AtomicLong(System.currentTimeMillis()); } public void open(final String filePath, String serializerType, Context serializerContext, final long rollInterval, final ScheduledExecutorService timedRollerPool, final FileWriterLinkedHashMap sfWriters) throws IOException { this.filePath = filePath; File file = new File(filePath + fileExtensionCounter + IN_USE_EXT); file.getParentFile().mkdirs(); outputStream = new BufferedOutputStream(new FileOutputStream(file)); logger.info("filename = " + file.getAbsolutePath()); serializer = EventSerializerFactory.getInstance(serializerType, serializerContext, outputStream); serializer.afterCreate(); if (rollInterval > 0) { Callable<Void> action = new Callable<Void>() { @Override public Void call() throws Exception { logger.debug( "Rolling file ({}): Roll scheduled after {} sec elapsed.", filePath + fileExtensionCounter + IN_USE_EXT, rollInterval); if (sfWriters.containsKey(filePath)) { sfWriters.remove(filePath); } close(); return null; } }; timedRollerPool.schedule(action, rollInterval, TimeUnit.SECONDS); } } public void append(Event event) throws IOException { serializer.write(event); } public boolean isBatchComplete() { return true; } public void flush() throws IOException { serializer.flush(); outputStream.flush(); } /** * Rename bucketPath file from .tmp to permanent location. */ private void renameBucket() { File srcPath = new File(filePath + fileExtensionCounter + IN_USE_EXT); File dstPath = new File(filePath + fileExtensionCounter); if (srcPath.exists()) { srcPath.renameTo(dstPath); logger.info("Renaming " + srcPath + " to " + dstPath); } } public synchronized void close() throws IOException, InterruptedException { if (outputStream != null) { outputStream.flush(); outputStream.close(); } renameBucket(); } }

FileWriterLinkedHashMap.java

package org.apache.flume.sink; import java.io.IOException; import java.util.LinkedHashMap; import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class FileWriterLinkedHashMap extends LinkedHashMap<String, BucketFileWriter> { private static final Logger logger = LoggerFactory .getLogger(FileWriterLinkedHashMap.class); private static final long serialVersionUID = -7860596835613215998L; private final int maxOpenFiles; public FileWriterLinkedHashMap(int maxOpenFiles) { super(16, 0.75f, true); // stock initial capacity/load, access this.maxOpenFiles = maxOpenFiles; } @Override protected boolean removeEldestEntry(Entry<String, BucketFileWriter> eldest) { if (size() > maxOpenFiles) { // If we have more that max open files, then close the last one // and // return true try { eldest.getValue().close(); } catch (IOException e) { logger.warn(eldest.getKey().toString(), e); } catch (InterruptedException e) { logger.warn(eldest.getKey().toString(), e); Thread.currentThread().interrupt(); } return true; } else { return false; } } }