HADOOP-promethues监控

安装jmx-exporter

$ wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar
mv jmx_prometheus_javaagent-0.13.0.jar /data/hdfs
$ for i in {2..5};do scp /data/hdfs/jmx_prometheus_javaagent-0.13.0.jar hadoop-test-$i:/data/hdfs/;done
$ cd ${HADOOP_HOME}

hdfs

$ vim /data/hdfs/nn.yaml
startDelaySeconds: 0
hostPort: 192.168.233.65:1234 #master为本机IP（一般可设置为localhost）；1234为想设置的jmx端口
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false

$ vim /data/hdfs/dn.yaml
---
startDelaySeconds: 0
hostPort: 192.168.233.65:1244 #master为本机IP（一般可设置为localhost）；1244为想设置的jmx端口（可设置为未被占用的端口）
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false

$ vim etc/hadoop/hadoop-env.sh
export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1234 -javaagent:/data/hdfs/jmx_prometheus_javaagent-0.13.0.jar=9211:/data/hdfs/nn.yaml"
export HADOOP_DATANODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1244 -javaagent:/data/hdfs/jmx_prometheus_javaagent-0.13.0.jar=9212:/data/hdfs/dn.yaml"

$ vim bin/hdfs
if [ "$COMMAND" = "namenode" ] ; then
  CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS $HADOOP_NAMENODE_JMX_OPTS" #添加namenode JMX环境变量
elif [ "$COMMAND" = "zkfc" ] ; then
  CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_ZKFC_OPTS"
elif [ "$COMMAND" = "secondarynamenode" ] ; then
  CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
elif [ "$COMMAND" = "datanode" ] ; then
  CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DATANODE_JMX_OPTS" #添加datanode jmx变量
  if [ "$starting_secure_dn" = "true" ]; then
    HADOOP_OPTS="$HADOOP_OPTS -jvm server $HADOOP_DATANODE_OPTS"
  else
    HADOOP_OPTS="$HADOOP_OPTS -server $HADOOP_DATANODE_OPTS"

$ stop-dfs.sh
$ start-dfs.sh

yarn

$ vim etc/hadoop/yarn-env.sh
export YARN_RESOURCEMANAGER_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1235 -javaagent:/data/hdfs/jmx_prometheus_javaagent-0.13.0.jar=9104:/data/hdfs/rm.yaml"
export YARN_NODEMANAGER_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1230 -javaagent:/data/hdfs/jmx_prometheus_javaagent-0.13.0.jar=9105:/data/hdfs/nm.yaml"

$ vim /data/hdfs/rm.yaml
startDelaySeconds: 0
hostPort: 192.168.233.65:1235 #master为本机IP（一般可设置为localhost）；1234为想设置的jmx端口
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false

$ vim /data/hdfs/nm.yaml
startDelaySeconds: 0
hostPort: 192.168.233.65:1230 #master为本机IP（一般可设置为localhost）；1244为想设置的jmx端口（可设置为未被占用的端口）
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false

$ vim bin/yarn
elif [ "$COMMAND" = "resourcemanager" ] ; then
  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
  CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
  CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
  CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
  YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS $YARN_RESOURCEMANAGER_OPTS" #增加列
  if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
...
...
...
elif [ "$COMMAND" = "nodemanager" ] ; then
  CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
  CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
  CLASSPATH=${CLASSPATH}:"$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
  CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
  YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
  if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then

$ stop-yarn.sh
$ start-yarn.sh

prometheus.yml

$ vim prometheus.yml
...
...
...
  - job_name: "NameNode"
    static_configs:
      - targets: ['192.168.233.65:9211','192.168.233.94:9211']
  - job_name: "DataNode"
    static_configs:
      - targets:
          - 192.168.233.65:9212
          - 192.168.233.17:9212
          - 192.168.233.157:9212
          - 192.168.233.238:9212
          - 192.168.233.94:9212
  - job_name: "ResourceManager"
    static_configs:
      - targets: ['192.168.233.65:9104','192.168.233.94:9104']
  - job_name: "NodeManager"
    static_configs:
      - targets:
          - 192.168.233.65:9105
          - 192.168.233.17:9105
          - 192.168.233.157:9105
          - 192.168.233.238:9105
          - 192.168.233.94:9105
$ systemctl restart prometheus