https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/
namenode.yaml
startDelaySeconds: 0
#master为本机IP(一般可设置为localhost);1234为想设置的jmx端口
hostPort: localhost:1234
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
datanode.yaml
hostPort: localhost:1235改这个端口号即可
startDelaySeconds: 0
#master为本机IP(一般可设置为localhost);1234为想设置的jmx端口
hostPort: localhost:1235
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
监控YARN组件
resourcemanager.yaml
…
nodemanager.yaml
…
路径 hadoop-2.7.7/etc/hadoop/
hadoop.env.sh 添加
export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1234 -javaagent:/home/monitor/jmx_exporter/jmx_prometheus_javaagent-0.15.0.jar=9201:/home/monitor/jmx_exporter/namenode.yaml"
export HADOOP_DATANODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1235 -javaagent:/home/monitor/jmx_exporter/jmx_prometheus_javaagent-0.15.0.jar=9202:/home/monitor/jmx_exporter/datanode.yaml"
注意改端口号,每个角色不能冲突
export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1234 -javaagent:/home/monitor/jmx_exporter/jmx_prometheus_javaagent-0.15.0.jar=9201:/home/monitor/jmx_exporter/namenode.yaml"
yarn.env.sh 添加
export YARN_NODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1237 -javaagent:/home/monitor/jmx_exporter/jmx_prometheus_javaagent-0.15.0.jar=9204:/home/monitor/jmx_exporter/nodemanager.yaml"
export YARN_RE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1236 -javaagent:/home/monitor/jmx_exporter/jmx_prometheus_javaagent-0.15.0.jar=9203:/home/monitor/jmx_exporter/resourcemanager.yaml"
路径hadoop-2.7.7/bin/
hdfs
if [ "$COMMAND" = "namenode" ] ; then
CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS $HADOOP_NAMENODE_JMX_OPTS"
elif [ "$COMMAND" = "zkfc" ] ; then
CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_ZKFC_OPTS"
elif [ "$COMMAND" = "secondarynamenode" ] ; then
CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
elif [ "$COMMAND" = "datanode" ] ; then
CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
if [ "$starting_secure_dn" = "true" ]; then
HADOOP_OPTS="$HADOOP_OPTS -jvm server $HADOOP_DATANODE_OPTS HADOOP_DATANODE_JMX_OPTS"
else
HADOOP_OPTS="$HADOOP_OPTS -server $HADOOP_DATANODE_OPTS HADOOP_DATANODE_JMX_OPTS"
fi
yarn
找到相关位置加入env里的变量
YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS $YARN_RE_JMX_OPTS"
elif [ "$COMMAND" = "resourcemanager" ] ; then
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS $YARN_RE_JMX_OPTS"
if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
fi
elif [ "$COMMAND" = "nodemanager" ] ; then
CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS $YARN_NODE_JMX_OPTS"
if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then
JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m"
fi
......
端口号
export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1234 -javaagent:/home/monitor/jmx_exporter/jmx_prometheus_javaagent-0.15.0.jar=9201:/home/monitor/jmx_exporter/namenode.yaml"
- job_name: 'namenode'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ['localhost:9201']
- job_name: 'datanode1'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ['node1:9202']
......
安装golang
https://studygolang.com/dl
下载tar.gz解压,配置环境变量即可。环境变量名 $GOPATH
安装glide
curl https://glide.sh/get | sh
安装git
yum install git-all
下载hadoop_exporter
https://github.com/Datatamer/hadoop_exporter
编译安装exporter
glide install
go build namenode_exporter.go
go build resourcemanager_exporter.go
[root@master hadoop_exporter-master]# glide install
[WARN] The name listed in the config file (Datatamer/hadoop_exporter) does not match the current location (.)
[WARN] Lock file may be out of date. Hash check of YAML failed. You may need to run 'update'
[INFO] Downloading dependencies. Please wait...
[INFO] --> Fetching github.com/beorn7/perks
[INFO] --> Fetching github.com/prometheus/log
[INFO] --> Fetching github.com/prometheus/procfs
[INFO] --> Fetching github.com/golang/protobuf
[INFO] --> Fetching github.com/prometheus/client_model
[INFO] --> Fetching github.com/Sirupsen/logrus
[INFO] --> Fetching golang.org/x/sys
[INFO] --> Fetching github.com/prometheus/client_golang
[INFO] --> Fetching github.com/matttproud/golang_protobuf_extensions
[INFO] --> Fetching github.com/prometheus/common
[WARN] Unable to checkout github.com/golang/protobuf
[ERROR] Update failed for github.com/golang/protobuf: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-golang-protobuf'...
fatal: unable to access 'https://github.com/golang/protobuf/': TCP connection reset by peer
: exit status 128
[WARN] Unable to checkout github.com/Sirupsen/logrus
[ERROR] Update failed for github.com/Sirupsen/logrus: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-Sirupsen-logrus'...
error: RPC failed; result=35, HTTP code = 0
fatal: The remote end hung up unexpectedly
: exit status 128
[WARN] Unable to checkout github.com/matttproud/golang_protobuf_extensions
[ERROR] Update failed for github.com/matttproud/golang_protobuf_extensions: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-matttproud-golang_protobuf_extensions'...
error: RPC failed; result=35, HTTP code = 0
fatal: The remote end hung up unexpectedly
: exit status 128
[WARN] Unable to checkout github.com/beorn7/perks
[ERROR] Update failed for github.com/beorn7/perks: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-beorn7-perks'...
fatal: unable to access 'https://github.com/beorn7/perks/': TCP connection reset by peer
: exit status 128
[WARN] Unable to checkout github.com/prometheus/client_model
[ERROR] Update failed for github.com/prometheus/client_model: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-prometheus-client_model'...
fatal: unable to access 'https://github.com/prometheus/client_model/': TCP connection reset by peer
: exit status 128
[WARN] Unable to checkout github.com/prometheus/common
[ERROR] Update failed for github.com/prometheus/common: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-prometheus-common'...
error: RPC failed; result=35, HTTP code = 0
fatal: The remote end hung up unexpectedly
: exit status 128
[WARN] Unable to checkout golang.org/x/sys
[ERROR] Update failed for golang.org/x/sys: Cannot detect VCS
[ERROR] Failed to install: Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-golang-protobuf'...
fatal: unable to access 'https://github.com/golang/protobuf/': TCP connection reset by peer
: exit status 128
Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-Sirupsen-logrus'...
error: RPC failed; result=35, HTTP code = 0
fatal: The remote end hung up unexpectedly
: exit status 128
Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-matttproud-golang_protobuf_extensions'...
error: RPC failed; result=35, HTTP code = 0
fatal: The remote end hung up unexpectedly
: exit status 128
Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-beorn7-perks'...
fatal: unable to access 'https://github.com/beorn7/perks/': TCP connection reset by peer
: exit status 128
Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-prometheus-client_model'...
fatal: unable to access 'https://github.com/prometheus/client_model/': TCP connection reset by peer
: exit status 128
Unable to get repository: Cloning into '/root/.glide/cache/src/https-github.com-prometheus-common'...
error: RPC failed; result=35, HTTP code = 0
fatal: The remote end hung up unexpectedly
: exit status 128
Cannot detect VCS