flume 1.9 源码阅读(三)

Flume-NG Configuration

上一节，看完了flume启动脚本，其中里面解析了各种用户传入的参数，并将其作为参数传给了 org.apache.flume.node.Application。 本节从 Application 入口类了解一下 flume-ng 如何启动。以及 flume-ng-configuration 模块是如何解析配置文件的。

org.apache.flume.node.Application

package org.apache.flume.node;

import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.flume.Channel;
import org.apache.flume.Constants;
import org.apache.flume.Context;
import org.apache.flume.SinkRunner;
import org.apache.flume.SourceRunner;
import org.apache.flume.instrumentation.MonitorService;
import org.apache.flume.instrumentation.MonitoringType;
import org.apache.flume.lifecycle.LifecycleAware;
import org.apache.flume.lifecycle.LifecycleState;
import org.apache.flume.lifecycle.LifecycleSupervisor;
import org.apache.flume.lifecycle.LifecycleSupervisor.SupervisorPolicy;
import org.apache.flume.util.SSLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;

public class Application {
  
  private static final Logger logger = LoggerFactory
      .getLogger(Application.class);

  public static final String CONF_MONITOR_CLASS = "flume.monitoring.type";
  public static final String CONF_MONITOR_PREFIX = "flume.monitoring.";

  private final List<LifecycleAware> components;
  private final LifecycleSupervisor supervisor;
  private MaterializedConfiguration materializedConfiguration;
  private MonitorService monitorServer;
  private final ReentrantLock lifecycleLock = new ReentrantLock();

  public Application() {
    this(new ArrayList<LifecycleAware>(0));
  }

  public Application(List<LifecycleAware> components) {
    this.components = components;
    supervisor = new LifecycleSupervisor();
  }

  public void start() {
    lifecycleLock.lock();
    try {
      for (LifecycleAware component : components) {
        supervisor.supervise(component,
            new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      }
    } finally {
      lifecycleLock.unlock();
    }
  }

  @Subscribe
  public void handleConfigurationEvent(MaterializedConfiguration conf) {
    try {
      lifecycleLock.lockInterruptibly();
      stopAllComponents();
      startAllComponents(conf);
    } catch (InterruptedException e) {
      logger.info("Interrupted while trying to handle configuration event");
      return;
    } finally {
      // If interrupted while trying to lock, we don't own the lock, so must not attempt to unlock
      if (lifecycleLock.isHeldByCurrentThread()) {
        lifecycleLock.unlock();
      }
    }
  }

  public void stop() {
    lifecycleLock.lock();
    stopAllComponents();
    try {
      supervisor.stop();
      if (monitorServer != null) {
        monitorServer.stop();
      }
    } finally {
      lifecycleLock.unlock();
    }
  }

  private void stopAllComponents() {
    if (this.materializedConfiguration != null) {
      logger.info("Shutting down configuration: {}", this.materializedConfiguration);
      for (Entry<String, SourceRunner> entry :
           this.materializedConfiguration.getSourceRunners().entrySet()) {
        try {
          logger.info("Stopping Source " + entry.getKey());
          supervisor.unsupervise(entry.getValue());
        } catch (Exception e) {
          logger.error("Error while stopping {}", entry.getValue(), e);
        }
      }

      for (Entry<String, SinkRunner> entry :
           this.materializedConfiguration.getSinkRunners().entrySet()) {
        try {
          logger.info("Stopping Sink " + entry.getKey());
          supervisor.unsupervise(entry.getValue());
        } catch (Exception e) {
          logger.error("Error while stopping {}", entry.getValue(), e);
        }
      }

      for (Entry<String, Channel> entry :
           this.materializedConfiguration.getChannels().entrySet()) {
        try {
          logger.info("Stopping Channel " + entry.getKey());
          supervisor.unsupervise(entry.getValue());
        } catch (Exception e) {
          logger.error("Error while stopping {}", entry.getValue(), e);
        }
      }
    }
    if (monitorServer != null) {
      monitorServer.stop();
    }
  }

  private void startAllComponents(MaterializedConfiguration materializedConfiguration) {
    logger.info("Starting new configuration:{}", materializedConfiguration);

    this.materializedConfiguration = materializedConfiguration;

    for (Entry<String, Channel> entry :
        materializedConfiguration.getChannels().entrySet()) {
      try {
        logger.info("Starting Channel " + entry.getKey());
        supervisor.supervise(entry.getValue(),
            new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      } catch (Exception e) {
        logger.error("Error while starting {}", entry.getValue(), e);
      }
    }

    /*
     * Wait for all channels to start.
     */
    for (Channel ch : materializedConfiguration.getChannels().values()) {
      while (ch.getLifecycleState() != LifecycleState.START
          && !supervisor.isComponentInErrorState(ch)) {
        try {
          logger.info("Waiting for channel: " + ch.getName() +
              " to start. Sleeping for 500 ms");
          Thread.sleep(500);
        } catch (InterruptedException e) {
          logger.error("Interrupted while waiting for channel to start.", e);
          Throwables.propagate(e);
        }
      }
    }

    for (Entry<String, SinkRunner> entry : materializedConfiguration.getSinkRunners().entrySet()) {
      try {
        logger.info("Starting Sink " + entry.getKey());
        supervisor.supervise(entry.getValue(),
            new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      } catch (Exception e) {
        logger.error("Error while starting {}", entry.getValue(), e);
      }
    }

    for (Entry<String, SourceRunner> entry :
         materializedConfiguration.getSourceRunners().entrySet()) {
      try {
        logger.info("Starting Source " + entry.getKey());
        supervisor.supervise(entry.getValue(),
            new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START);
      } catch (Exception e) {
        logger.error("Error while starting {}", entry.getValue(), e);
      }
    }

    this.loadMonitoring();
  }

  @SuppressWarnings("unchecked")
  private void loadMonitoring() {
    Properties systemProps = System.getProperties();
    Set<String> keys = systemProps.stringPropertyNames();
    try {
      if (keys.contains(CONF_MONITOR_CLASS)) {
        String monitorType = systemProps.getProperty(CONF_MONITOR_CLASS);
        Class<? extends MonitorService> klass;
        try {
          //Is it a known type?
          klass = MonitoringType.valueOf(
              monitorType.toUpperCase(Locale.ENGLISH)).getMonitorClass();
        } catch (Exception e) {
          //Not a known type, use FQCN
          klass = (Class<? extends MonitorService>) Class.forName(monitorType);
        }
        this.monitorServer = klass.newInstance();
        Context context = new Context();
        for (String key : keys) {
          if (key.startsWith(CONF_MONITOR_PREFIX)) {
            context.put(key.substring(CONF_MONITOR_PREFIX.length()),
                systemProps.getProperty(key));
          }
        }
        monitorServer.configure(context);
        monitorServer.start();
      }
    } catch (Exception e) {
      logger.warn("Error starting monitoring. "
          + "Monitoring might not be available.", e);
    }

  }

  public static void main(String[] args) {

    try {
      SSLUtil.initGlobalSSLParameters();
	  
      Options options = new Options();

      Option option = new Option("n", "name", true, "the name of this agent");
      option.setRequired(true);
      options.addOption(option);

      option = new Option("f", "conf-file", true,
          "specify a config file (required if -z missing)");
      option.setRequired(false);
      options.addOption(option);

      option = new Option(null, "no-reload-conf", false,
          "do not reload config file if changed");
      options.addOption(option);

      // Options for Zookeeper
      option = new Option("z", "zkConnString", true,
          "specify the ZooKeeper connection to use (required if -f missing)");
      option.setRequired(false);
      options.addOption(option);

      option = new Option("p", "zkBasePath", true,
          "specify the base path in ZooKeeper for agent configs");
      option.setRequired(false);
      options.addOption(option);

      option = new Option("h", "help", false, "display help text");
      options.addOption(option);

      CommandLineParser parser = new GnuParser();
      CommandLine commandLine = parser.parse(options, args);

      if (commandLine.hasOption('h')) {
        new HelpFormatter().printHelp("flume-ng agent", options, true);
        return;
      }

      String agentName = commandLine.getOptionValue('n');
      boolean reload = !commandLine.hasOption("no-reload-conf");

      boolean isZkConfigured = false;
      if (commandLine.hasOption('z') || commandLine.hasOption("zkConnString")) {
        isZkConfigured = true;
      }

      Application application;
      if (isZkConfigured) {
        // get options
        String zkConnectionStr = commandLine.getOptionValue('z');
        String baseZkPath = commandLine.getOptionValue('p');

        if (reload) {
          EventBus eventBus = new EventBus(agentName + "-event-bus");
          List<LifecycleAware> components = Lists.newArrayList();
          PollingZooKeeperConfigurationProvider zookeeperConfigurationProvider =
              new PollingZooKeeperConfigurationProvider(
                  agentName, zkConnectionStr, baseZkPath, eventBus);
          components.add(zookeeperConfigurationProvider);
          application = new Application(components);
          eventBus.register(application);
        } else {
          StaticZooKeeperConfigurationProvider zookeeperConfigurationProvider =
              new StaticZooKeeperConfigurationProvider(
                  agentName, zkConnectionStr, baseZkPath);
          application = new Application();
          application.handleConfigurationEvent(zookeeperConfigurationProvider.getConfiguration());
        }
      } else {
        File configurationFile = new File(commandLine.getOptionValue('f'));

        /*
         * The following is to ensure that by default the agent will fail on
         * startup if the file does not exist.
         */
        if (!configurationFile.exists()) {
          // If command line invocation, then need to fail fast
          if (System.getProperty(Constants.SYSPROP_CALLED_FROM_SERVICE) ==
              null) {
            String path = configurationFile.getPath();
            try {
              path = configurationFile.getCanonicalPath();
            } catch (IOException ex) {
              logger.error("Failed to read canonical path for file: " + path,
                  ex);
            }
            throw new ParseException(
                "The specified configuration file does not exist: " + path);
          }
        }
        List<LifecycleAware> components = Lists.newArrayList();

        if (reload) {
          EventBus eventBus = new EventBus(agentName + "-event-bus");
          PollingPropertiesFileConfigurationProvider configurationProvider =
              new PollingPropertiesFileConfigurationProvider(
                  agentName, configurationFile, eventBus, 30);
          components.add(configurationProvider);
          application = new Application(components);
          eventBus.register(application);
        } else {
          PropertiesFileConfigurationProvider configurationProvider =
              new PropertiesFileConfigurationProvider(agentName, configurationFile);
          application = new Application();
          application.handleConfigurationEvent(configurationProvider.getConfiguration());
        }
      }
      application.start();

      final Application appReference = application;
      Runtime.getRuntime().addShutdownHook(new Thread("agent-shutdown-hook") {
        @Override
        public void run() {
          appReference.stop();
        }
      });

    } catch (Exception e) {
      logger.error("A fatal error occurred while running. Exception follows.", e);
    }
  }
}

flume 一共有两种读取配置文件方式，一种是通过-z / --zkConnString 从zookeeper中读取配置文件，一种是通过-f / --conf-file 从本地文件系统读取配置文件，配置文件格式遵循了 java properties 文件的格式。我们可以将多个 agent 定义在同一个配置文件中, 配置文件中包含了所需要的 source, sink, channel 组件，并且还要说明他们是怎么连接到一起的。

flume 启动的时候有一个叫做 --no-reload-conf 的配置项，这个配置的作用是当flume配置文件变动时是否可以热加载修改后的配置文件，而不用重启flume。

这里我们从本地文件系统读取配置文件，主要代码如下：

		// 获取配置文件对象
		File configurationFile = new File(commandLine.getOptionValue('f'));
        /*
          * 首先检查一下配置文件是否存在，不存在的话直接抛异常
          */
         if (!configurationFile.exists()) {
             // If command line invocation, then need to fail fast
             if (System.getProperty(Constants.SYSPROP_CALLED_FROM_SERVICE) ==
                     null) {
                 String path = configurationFile.getPath();
                 try {
                     path = configurationFile.getCanonicalPath();
                 } catch (IOException ex) {
                     logger.error("Failed to read canonical path for file: " + path,
                             ex);
                 }
                 throw new ParseException(
                         "The specified configuration file does not exist: " + path);
             }
         }
         
         List<LifecycleAware> components = Lists.newArrayList();

         if (reload) {
             //谷歌 guava 事件总线有兴趣的可以了解下(我还没看) ，大概的意思就是 30 秒轮询一次配置文件
             EventBus eventBus = new EventBus(agentName + "-event-bus");
             PollingPropertiesFileConfigurationProvider configurationProvider =
                     new PollingPropertiesFileConfigurationProvider(
                             agentName, configurationFile, eventBus, 30);
             components.add(configurationProvider);
             application = new Application(components);
             eventBus.register(application);
         } else {
             PropertiesFileConfigurationProvider configurationProvider =
                     new PropertiesFileConfigurationProvider(agentName, configurationFile);
             application = new Application();
             application.handleConfigurationEvent(configurationProvider.getConfiguration());
         }

可以看到这里面解析配置文件由 PollingPropertiesFileConfigurationProvider 或者 PropertiesFileConfigurationProvider 两个类来完成，具体要看是否启用了 no-reload-conf 配置。但是最终都是要去实例化 FlumeConfiguration 这个类。

    public FlumeConfiguration(Map<String, String> properties) {
        agentConfigMap = new HashMap<>();
        errors = new LinkedList<>();
        // Construct the in-memory component hierarchy
        for (Entry<String, String> entry : properties.entrySet()) {
            if (!addRawProperty(entry.getKey(), entry.getValue())) {
                LOGGER.warn("Configuration property ignored: {} = {}", entry.getKey(), entry.getValue());
            }
        }
        // Now iterate thru the agentContext and create agent configs and add them
        // to agentConfigMap
        // validate and remove improperly configured components
        validateConfiguration();
    }

FlumeConfiguration 构造函数如上，主要是将配置文件中的 key , value 遍历出来，然后调用 addRawProperty 方法将他们添加进FlumeConfiguration的成员变量agentConfigMap中。

    private boolean addRawProperty(String rawName, String rawValue) {
        // Null names and values not supported
        if (rawName == null || rawValue == null) {
            addError("", AGENT_NAME_MISSING, ERROR);
            return false;
        }
        // Remove leading and trailing spaces
        String name = rawName.trim();
        String value = rawValue.trim();
        // Empty values are not supported
        if (value.isEmpty()) {
            addError(name, PROPERTY_VALUE_NULL, ERROR);
            return false;
        }
		//获取第一个"."(点)的位置
        int index = name.indexOf('.');

        // All configuration keys must have a prefix defined as agent name
        if (index == -1) {
            addError(name, AGENT_NAME_MISSING, ERROR);
            return false;
        }
		//获取 agent 名字
        String agentName = name.substring(0, index);

        // Agent name must be specified for all properties
        if (agentName.isEmpty()) {
            addError(name, AGENT_NAME_MISSING, ERROR);
            return false;
        }

		//获取 . 后面的字符串
        String configKey = name.substring(index + 1);

        // Configuration key must be specified for every property
        if (configKey.isEmpty()) {
            addError(name, PROPERTY_NAME_NULL, ERROR);
            return false;
        }
		//当agentConfigMap中不存在当前agent, 将解析出来的 agent 名字 添加到agentConfigMap中,同时也将flumeConfiguration中的AgentConfiguration对象也实例化出来
        AgentConfiguration aconf = agentConfigMap.get(agentName);

        if (aconf == null) {
            aconf = new AgentConfiguration(agentName, errors);
            agentConfigMap.put(agentName, aconf);
        }

        //最后还需要继续解析 agent 名字后面的配置项
        return aconf.addProperty(configKey, value);
    }

private boolean addProperty(String key, String value) {
//这里分别要判断配置项是属于哪一类: configfilter , sources, sinks, channels, sinkgroups
            // Check for configFilters
            //判断是不是 .configfilters = xxx 形式
            if (CONFIG_CONFIGFILTERS.equals(key)) {
                if (configFilters == null) {
                    configFilters = value;
                    return true;
                } else {
                    LOGGER.warn("Duplicate configfilter list specified for agent: {}", agentName);
                    addError(CONFIG_CONFIGFILTERS, DUPLICATE_PROPERTY, ERROR);
                    return false;
                }
            }
            // Check for sources
            //判断是不是 .sources = xxx 形式
            if (CONFIG_SOURCES.equals(key)) {
                if (sources == null) {
                    sources = value;
                    return true;
                } else {
                    LOGGER.warn("Duplicate source list specified for agent: {}", agentName);
                    addError(CONFIG_SOURCES, DUPLICATE_PROPERTY, ERROR);
                    return false;
                }
            }

            // Check for sinks
            //判断是不是 .sinks = xxx 形式
            if (CONFIG_SINKS.equals(key)) {
                if (sinks == null) {
                    sinks = value;
                    LOGGER.info("Added sinks: {} Agent: {}", sinks, agentName);
                    return true;
                } else {
                    LOGGER.warn("Duplicate sink list specfied for agent: {}", agentName);
                    addError(CONFIG_SINKS, DUPLICATE_PROPERTY, ERROR);
                    return false;
                }
            }

            // Check for channels
            //判断是不是 .channels = xxx 形式
            if (CONFIG_CHANNELS.equals(key)) {
                if (channels == null) {
                    channels = value;

                    return true;
                } else {
                    LOGGER.warn("Duplicate channel list specified for agent: {}", agentName);
                    addError(CONFIG_CHANNELS, DUPLICATE_PROPERTY, ERROR);
                    return false;
                }
            }

            // Check for sinkgroups
            //判断是不是 .sinkgroups = xxx 形式
            if (CONFIG_SINKGROUPS.equals(key)) {
                if (sinkgroups == null) {
                    sinkgroups = value;

                    return true;
                } else {
                    LOGGER.warn("Duplicate sinkgroup list specfied for agent: {}", agentName);
                    addError(CONFIG_SINKGROUPS, DUPLICATE_PROPERTY, ERROR);
                    return false;
                }
            }
		// 以上五种形式会直接将 . 后面的字符串传到 AgentConfiguration 数据结构中对应字段中
		// 以上五个组件 configfilter、sources、channels、sinks、sinkgroups 可以称为 component.

		//然后会调用 addAsXXXConfig 方法来检查这一行配置文件是以什么开头的，其实最后调用的都是
		//addComponentConfig方法。这个方法会将后面的参数传入到AgentConfiguration 的各种 context 的 Map中。
            if (addAsSourceConfig(key, value)
                    || addAsChannelValue(key, value)
                    || addAsSinkConfig(key, value)
                    || addAsSinkGroupConfig(key, value)
                    || addAsConfigFilterConfig(key, value)
            ) {
                return true;
            }

            LOGGER.warn("Invalid property specified: {}", key);
            addError(key, INVALID_PROPERTY, ERROR);
            return false;
        }

        private boolean addComponentConfig(
                String key, String value, String configPrefix, Map<String, Context> contextMap
        ) {
            ComponentNameAndConfigKey parsed = parseConfigKey(key, configPrefix);
            if (parsed != null) {
                String name = parsed.getComponentName().trim();
                LOGGER.info("Processing:{}", name);
                Context context = contextMap.get(name);

                if (context == null) {
                    LOGGER.debug("Created context for {}: {}", name, parsed.getConfigKey());
                    context = new Context();
                    contextMap.put(name, context);
                }
                context.put(parsed.getConfigKey(), value);
                return true;
            }

            return false;
        }

看到这里感觉 flume 配置文件就是按树状结构一层一层的去解析。这里有个小问题就是使用java Properties类读取 properties 文件，不是按顺序从头到尾。所以这里面就依靠各种名字去匹配对应关系，FlumeConfiguration的数据结构中体现为 Map agentConfigMap，以及AgentConfiguration中的一系列的名字后面会详细的看一下。