Flume-ng 数据发送速度限制

Flume-ng 数据发送速度限制

转自:http://blog.csdn.net/desilting/article/details/27081357

   按理说,应该在sink端限制数据的发送速度,但flume-ng提供了非常便利的interceptor模式,因此本文,就只是在source端简单的实现了对数据发送速度的限制。

[java]  view plain copy
  1. package com.xxx.flume.core.interceptor;  
  2.   
  3. import java.util.List;  
  4. import org.slf4j.Logger;  
  5. import org.apache.flume.Event;  
  6. import org.slf4j.LoggerFactory;  
  7. import org.apache.flume.Context;  
  8. import org.apache.flume.interceptor.Interceptor;  
  9.   
  10.   
  11. public class LimitInterceptor implements Interceptor {  
  12.     private static final Logger logger = LoggerFactory.getLogger(LimitInterceptor.class);  
  13.   
  14.     private static long KB = 1024L;  
  15.   
  16.     private long lastEventSentTick = System.nanoTime();  
  17.   
  18.     private long pastSentLength = 0L;  
  19.     private long max;  
  20.     private long timeCostPerCheck = 1000000000L;  
  21.   
  22.     private long headerSize = 0L;  
  23.   
  24.     private boolean flag = true;  
  25.   
  26.     private int num = 0;  
  27.   
  28.     public LimitInterceptor(long limitRate, long headerSize) {  
  29.         this.max = (limitRate * KB);  
  30.         this.headerSize = headerSize;  
  31.     }  
  32.   
  33.     public void initialize() {  
  34.     }  
  35.   
  36.     public Event intercept(Event event) {  
  37.         this.num += 1;  
  38.         if (this.pastSentLength > this.max) {  
  39.             long nowTick = System.nanoTime();  
  40.             long multiple = this.pastSentLength / this.max;  
  41.             long missedTime = multiple * this.timeCostPerCheck - (nowTick - this.lastEventSentTick);  
  42.             if (missedTime > 0L) {  
  43.                 try {  
  44.                     System.out.printf("Limit source send rate, headerLength:%d,pastSentLength:%d,lastEventSentTick:%d,
  45. sleepTime:%d, num:%d\n",  
  46.                         headerSize, pastSentLength, lastEventSentTick, missedTime / 1000000, num);  
  47.                     Thread.sleep(missedTime / 1000000L, (int) (missedTime % 1000000L));  
  48.                 } catch (InterruptedException e) {  
  49.                     e.printStackTrace();  
  50.                 }  
  51.             }  
  52.             this.num = 0;  
  53.             this.pastSentLength = 0L;  
  54.             this.lastEventSentTick = (nowTick + (missedTime > 0L ? missedTime : 0L));  
  55.         }  
  56.         this.pastSentLength += this.headerSize + event.getBody().length;  
  57.   
  58.         return event;  
  59.     }  
  60.   
  61.     public List<Event> intercept(List<Event> events) {  
  62.         for (Event event : events) {  
  63.             intercept(event);  
  64.         }  
  65.         return events;  
  66.     }  
  67.   
  68.     public void close() {  
  69.     }  
  70.   
  71.     public static class Builder implements Interceptor.Builder {  
  72.         private long limitRate;  
  73.         private long headerSize;  
  74.   
  75.         public Interceptor build() {  
  76.             return new LimitInterceptor(this.limitRate, this.headerSize);  
  77.         }  
  78.   
  79.         public void configure(Context context) {  
  80.             this.limitRate = context.getLong(Constants.LIMIT_RATE, Long.valueOf(Constants.DEFAULT_RATE)).longValue();  
  81.             this.headerSize = context.getLong(Constants.HEADER_SIZE, Long.valueOf(Constants.DEFAULT_SIZE)).longValue();  
  82.         }  
  83.   
  84.         public static class Constants {  
  85.             public static long DEFAULT_RATE = 500L;  
  86.             public static long DEFAULT_SIZE = 16L;  
  87.             public static String LIMIT_RATE = "limitRate";  
  88.             public static String HEADER_SIZE = "headerSize";  
  89.         }  
  90.     }  
  91. }  

打包后扔到flume-ng的lib目录下,然后在flume-conf配置文件,增加流速拦截器:

[html]  view plain copy
  1. agent.sources.seqGenSrc.interceptors = limitrate  
  2. agent.sources.seqGenSrc.interceptors.limitrate.type = com.xxx.flume.core.interceptor.LimitInterceptor$Builder  
  3. agent.sources.seqGenSrc.interceptors.limitrate.limitRate = 500  
  4. agent.sources.seqGenSrc.interceptors.limitrate.headerSize = 8  

这里限制发送速度500kb/s(可以为不同的source设置不同的发送速度),设置每个event头大小约为8字节。

发送端输出信息:

[plain]  view plain copy
  1. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649436601991827,sleepTime:929, num:3437  
  2. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649437601991827,sleepTime:929, num:3437  
  3. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649438601991827,sleepTime:929, num:3437  
  4. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649439601991827,sleepTime:927, num:3437  
  5. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649440601991827,sleepTime:928, num:3437  
  6. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649441601991827,sleepTime:928, num:3437  
  7. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649442601991827,sleepTime:928, num:3437  
  8. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649443601991827,sleepTime:925, num:3437  
  9. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649444601991827,sleepTime:929, num:3437  
  10. Limit source send rate, headerLength:8,pastSentLength:512025,lastEventSentTick:79649445601991827,sleepTime:889, num:3421  
  11. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649446601991827,sleepTime:924, num:3437  
  12. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649447601991827,sleepTime:927, num:3437  
  13. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSentTick:79649448601991827,sleepTime:928, num:3437  
  14. Limit source send rate, headerLength:8,pastSentLength:512113,lastEventSen

你可能感兴趣的:(Flume-ng 数据发送速度限制)