https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/examples/demo
https://pkg.go.dev/github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor#section-readme
https://github.com/open-telemetry/opentelemetry-demo/blob/main/docker-compose.yml
https://github.com/laziobird/otel-collector-java/
服务器IP | 部暑角色 |
---|---|
192.168.11.100 | elasticsearch 、jaeger-all-in-one、otel-collector、 hotrod、spark dependencies、promethues grafana |
jaeger组件介绍:
elasticsearch : jaeger 存储器
jaeger-all-in-one:
spark denpendencies: 对spans进行集合,生成依赖图
promethus: 监控
grafana:图显
otel-collector-contrib:
一、docker部暑
。。。
二、elasticsearch
2.1、集群证书生成,生成elastic-certificates.p12证书(此步要手动执行确认)
```bash
mkdir -p /data/elasticsearch/{config,logs,data}/
mkdir -p /data/elasticsearch/config/certs/
chown 1000:root /data/elasticsearch/{config,logs,data}
docker run -it --rm \
-v /data/elasticsearch/config/:/usr/share/elasticsearch/config/ \
elasticsearch:7.17.6 bash
#以下需要手动执行
bin/elasticsearch-certutil ca -s --pass '' --days 10000 --out elastic-stack-ca.p12
bin/elasticsearch-certutil cert -s --ca-pass '' --pass '' --days 5000 --ca elastic-stack-ca.p12 --out elastic-certificates.p12
bin/elasticsearch-certutil cert -s --ca-pass '' --pass '' --days 5000 --ca elastic-stack-ca.p12 -name 'CN=szgd,OU=elastic Team,DC=digitalgd,DC=com' --out elastic-https.p12 --ip '192.168.11.100,127.0.0.1' --dns *.digitalgd.com
#创建https证书(filebeat、kibana使用)
#./bin/elasticsearch-certutil cert -s --ca-pass '' --pass '' --days 5000 --ca elastic-stack-ca.p12 -name "CN=szgd,OU=elastic Team,DC=digitalgd,DC=com" --out client.p12 --ip "192.168.11.100,192.168.11.101,192.168.11.102,127.0.0.1" --dns "*.digitalgd.com"
openssl pkcs12 -in elastic-https.p12 -nocerts -nodes -password pass: > client.key
openssl pkcs12 -in elastic-https.p12 -clcerts -nokeys -password pass: > client.cer
openssl pkcs12 -in elastic-https.p12 -cacerts -nokeys -chain -password pass: > client-ca.cer
mv elastic-* config/certs
mv client* config/certs
chown -R 1000:root config
exit
2.2 准备elasticsearch.yml
mkdir -p /data/elasticsearch/{config,data}
cat > /data/elasticsearch/config/elasticsearch.yml << 'EOF'
cluster.name: smartgate-cluster
discovery.seed_hosts: 192.168.11.100
cluster.initial_master_nodes: 192.168.11.100
network.host: 192.168.11.100
#增加了写队列的大小
thread_pool.write.queue_size: 1000
#锁定内存
bootstrap.memory_lock: true
bootstrap.system_call_filter: false
xpack.license.self_generated.type: basic
xpack.ml.enabled: false
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: "certificate"
xpack.security.transport.ssl.keystore.path: "certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "certs/elastic-certificates.p12"
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: certs/elastic-https.p12
xpack.security.http.ssl.truststore.path: certs/elastic-https.p12
xpack.security.http.ssl.client_authentication: optional
xpack.security.authc.realms.pki.pki1.order: 1
node.roles: ['master','data','ingest','remote_cluster_client']
node.attr.box_type: hot
node.attr.disk_type: sas
node.name: 192.168.11.100
http.port: 9200
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization
http.cors.enabled: true
http.host: "192.168.11.100,127.0.0.1"
transport.host: "192.168.11.100,127.0.0.1"
#elastic agent 启动需要
#xpack.security.authc.api_key.enabled: true
ingest.geoip.downloader.enabled: false
EOF
cat >/data/elasticsearch/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
dockerd --iptables=false >/dev/nul 2>&1 &
sleep 1
docker start elasticsearch >/dev/nul 2>&1
if [ "$?" == "0" ]
then
docker rm elasticsearch -f
fi
sleep 1
docker start elasticsearch >/dev/nul 2>&1
if [ "$?" != "0" ]
then
echo "run elasticsearch"
docker run -d \
--restart=always \
--name elasticsearch \
--network host \
--privileged \
--ulimit memlock=-1:-1 \
--ulimit nofile=65536:65536 \
-e ELASTIC_PASSWORD=xxxxxxxx \
-e KIBANA_PASSWORD=xxxxxxxx \
-e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
-v /etc/localtime:/etc/localtime \
-v `pwd`/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
-v `pwd`/config/certs/:/usr/share/elasticsearch/config/certs \
-v `pwd`/data/:/usr/share/elasticsearch/data/ \
-v `pwd`/logs/:/usr/share/elasticsearch/logs/ \
elasticsearch:7.17.6
fi
EOF
bash /data/elasticsearch/start.sh
2.3 验证es
curl -u elastic:xxxxxxxx https://192.168.11.100:9200/ -k
{
"name" : "192.168.11.100",
"cluster_name" : "smartgate-cluster",
"cluster_uuid" : "WXsvx4dpTVi5o27Zdlh5xg",
"version" : {
"number" : "7.17.6",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "f65e9d338dc1d07b642e14a27f338990148ee5b6",
"build_date" : "2022-08-23T11:08:48.893373482Z",
"build_snapshot" : false,
"lucene_version" : "8.11.1",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
#开放firewalld端口
firewall-cmd --permanent --add-port=9200/tcp
firewall-cmd --permanent --add-port=9300/tcp
firewall-cmd --reload
三、prometheus
#创建prometheus工作目录
mkdir /data/prometheus/{data,conf,conf/rules,conf/sd_config} -p
chown -R 65534:65534 /data/prometheus/data
#promethes配置文件
cat > /data/prometheus/conf/prometheus.yml << 'EOF'
global:
evaluation_interval: 30s
scrape_interval: 5s
scrape_configs:
- job_name: otel
honor_labels: true
static_configs:
- targets:
- '192.168.11.100:9464'
- job_name: otel-collector
static_configs:
- targets:
- '192.168.11.100:8888'
EOF
#启动脚本
cat > /data/prometheus/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f prometheus
docker run -d \
--name prometheus \
--restart=always \
-p 9090:9090 \
-v /data/prometheus/conf/prometheus.yml:/etc/prometheus/prometheus.yml \
-v /data/prometheus/conf/rules:/etc/prometheus/rules \
-v /data/prometheus/conf/sd_config:/etc/prometheus/sd_config \
-v /data/prometheus/data:/data/prometheus \
-v /etc/localtime:/etc/localtime:ro \
prom/prometheus:v2.28.0 \
--web.read-timeout=5m \
--config.file=/etc/prometheus/prometheus.yml \
--storage.tsdb.path=/data/prometheus \
--web.max-connections=512 \
--storage.tsdb.retention=30d \
--query.timeout=2m \
--web.enable-lifecycle \
--web.listen-address=:9090 \
--web.enable-admin-api \
--enable-feature=remote-write-receiver
EOF
bash /data/prometheus/start.sh
#开放firewalld端口
firewall-cmd --permanent --add-port=9090/tcp
firewall-cmd --reload
验证地址
http://192.168.11.100:9090
四、grafana
path=/data/grafana
mkdir ${path}/data -p
chown -R 472.472 ${path}/data
cat >${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f grafana
docker run -d \
--name=grafana \
--restart always \
--user 472 \
-p 3000:3000 \
-e "GF_SECURITY_ADMIN_USER=admin" \
-e "GF_SECURITY_ADMIN_PASSWORD=xxxxxxxx" \
-e "GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-clickhouse-datasource" \
-e "GF_SERVER_SERVE_FROM_SUB_PATH=true" \
-e "GF_SERVER_ROOT_URL=http://192.168.11.100:3000/grafana/" \
-v /etc/localtime:/etc/localtime:ro \
-v /data/grafana/data:/var/lib/grafana \
grafana/grafana:9.3.6
EOF
bash ${path}/start.sh
http://192.168.11.100:3000/grafana/
帐密: admin / xxxxxxxx
四、jaeger(all-in-one)
path=/data/jaeger
mkdir ${path} -p
#把elasticsearch 生成证书copy到 jaeger目录下
cp /data/elasticsearch/config/certs /data/jaeger/ -rf
cat >${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f jaeger
docker run -d \
--name jaeger \
-e COLLECTOR_OTLP_ENABLED=true \
-p 5775:5775/udp \
-p 6831:6831/udp \
-p 6832:6832/udp \
-p 16686:16686 \
-p 14250:14250 \
-v /etc/localtime:/etc/localtime \
-v `pwd`/certs:/go/certs \
-e SPAN_STORAGE_TYPE=elasticsearch \
-e ES_USERNAME=elastic \
-e ES_PASSWORD=xxxxxxxx \
-e ES_SERVER_URLS=https://192.168.11.100:9200 \
-e ES_TLS_ENABLED=true \
-e ES_TLS_CA=/go/certs/client-ca.cer \
-e METRICS_STORAGE_TYPE=prometheus \
-e PROMETHEUS_SERVER_URL=http://192.168.11.100:9090 \
-e PROMETHEUS_TLS_ENABLED=false \
jaegertracing/all-in-one:1.45
EOF
bash ${path}/start.sh
#验证地址
http://192.168.11.100:16686/search
五、otel-collector
path=/data/otel
mkdir ${path} -p
cat >${path}/otel-config.yaml << 'EOF'
receivers:
jaeger:
protocols:
thrift_http:
endpoint: :14268
# Dummy receiver that's never used, because a pipeline is required to have one.
otlp/spanmetrics:
protocols:
grpc:
endpoint: "localhost:12345"
otlp:
protocols:
grpc:
endpoint: :4317
http:
endpoint: :4318
prometheus:
config:
scrape_configs:
- job_name: 'ratelimiter'
scrape_interval: 15s
static_configs:
- targets: [ "0.0.0.0:8889" ]
exporters:
jaeger:
endpoint: 192.168.11.100:14250
tls:
insecure: true
prometheus:
endpoint: "0.0.0.0:8889"
prometheusremotewrite:
endpoint: http://192.168.11.100:9090/api/v1/write
target_info:
enabled: true
processors:
batch:
spanmetrics:
metrics_exporter: prometheus
latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
dimensions:
- name: http.method
default: GET
- name: http.status_code
dimensions_cache_size: 1000
aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"
metrics_flush_interval: 15s
extensions:
health_check:
pprof:
endpoint: :1888
zpages:
endpoint: :55679
service:
extensions: [pprof, zpages, health_check]
pipelines:
traces:
receivers: [jaeger,otlp]
processors: [spanmetrics, batch]
exporters: [jaeger]
metrics/spanmetrics:
receivers: [otlp/spanmetrics]
exporters: [prometheus]
metrics:
receivers: [prometheus]
exporters: [prometheusremotewrite]
EOF
cat >${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f otel
docker run -itd \
--restart=always \
--name otel \
-p 1888:1888 \
-p 8888:8888 \
-p 8889:8889 \
-p 13133:13133 \
-p 4317:4317 \
-p 55679:55679 \
-p 12345:12345 \
-p 14268:14268 \
-v /etc/localtime:/etc/localtime \
-v `pwd`/otel-config.yaml:/etc/otel-collector-config.yaml \
otel/opentelemetry-collector-contrib:0.78.0 \
--config=/etc/otel-collector-config.yaml
# - "1888:1888" # pprof extension
# - "8888:8888" # Prometheus metrics exposed by the collector
# - "8889:8889" # Prometheus exporter metrics
# - "13133:13133" # health_check extension
# - "4317:4317" # OTLP gRPC receiver
# - "55679:55679" # zpages extension
# - "12345:12345" # zpages extension
EOF
bash ${path}/start.sh
六、spark dependencies(与elasticsearch 集成)
6.1 生成cacerts证书,提供给dependencies使用
path=/data/jaeger
mkdir ${path} -p
cd ${path}
#进入容器
docker run --rm -it --entrypoint bash -u root -v `pwd`/certs/:/certs/ jaegertracing/spark-dependencies
#获取elasticsearch的https证书
openssl s_client -connect 192.168.11.100:9200 < /dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' >> ca.crt
#把elasticsearch的https证书导入cacerts
echo 'yes'|keytool -import -alias elasticsearch -keystore /certs/cacerts -storepass "123456" -file ca.crt
#退出容器
exit
cat > ${path}/denpendencies.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker run --rm -it \
-e STORAGE=elasticsearch \
-e ES_NODES=https://192.168.11.100:9200 \
-e ES_TIME_RANGE=24h \
-e ES_USERNAME=elastic \
-e ES_PASSWORD=xxxxxxxx \
-v `pwd`/certs/cacerts:/usr/local/openjdk-8/lib/security/cacerts \
-v /etc/localtime:/etc/localtime \
jaegertracing/spark-dependencies
EOF
bash ${path}/denpendencies.sh
demo1
path=/data/demo
mkdir ${path} -p
cd ${path}
cat >${path}/start.sh << 'EOF'
#!/bin/bash
docker rm -f demo-server
docker run -d \
--restart=always \
--name demo-server \
-p 7080:7080 \
-e OTEL_EXPORTER_OTLP_ENDPOINT=192.168.11.100:4317 \
zengxiangbang/demo_demo-server
docker rm -f demo-client
docker run -d \
--restart=always \
--name demo-client \
--net host \
-e OTEL_EXPORTER_OTLP_ENDPOINT=192.168.11.100:4317 \
-e DEMO_SERVER_ENDPOINT=http://192.168.11.100:7080/hello \
zengxiangbang/demo_demo-client
EOF
bash ${path}/start.sh
dem2
path=/data/hotrod
mkdir ${path} -p
cat >${path}/start.sh << 'EOF'
#!/bin/bash
docker rm -f hotrod
cd `dirname $0`
docker run -d \
--restart=always \
--name=hotrod \
-v /etc/localtime:/etc/localtime \
-p 8080:8080 \
-p 8081:8081 \
-p 8083:8083 \
-e OTEL_EXPORTER_JAEGER_ENDPOINT=http://192.168.11.100:14268/api/traces \
jaegertracing/example-hotrod:1.45 all -m prometheus
#-e OTEL_EXPORTER_JAEGER_AGENT_HOST=192.168.11.100 \
#-e OTEL_EXPORTER_JAEGER_AGENT_PORT=6831 \
EOF
bash ${path}/start.sh
#开放firewalld端口
firewall-cmd --permanent --add-port=8080/tcp
firewall-cmd --permanent --add-port=8081/tcp
firewall-cmd --permanent --add-port=8083/tcp
firewall-cmd --reload
#访问hotrod
http://192.168.11.100:8081/
demo3、自动生成
docker run --net host --rm yurishkuro/microsim -d 60s
spanmetricsconnectors otel-config.yaml 配置方法 ,未来偏向于用这种方法
cat >${path}/otel-config.yaml << 'EOF'
receivers:
jaeger:
protocols:
thrift_http:
endpoint: :14278
otlp:
protocols:
grpc:
endpoint: :4317
http:
endpoint: :4318
exporters:
jaeger:
endpoint: 192.168.11.100:14250
tls:
insecure: true
prometheus:
endpoint: "0.0.0.0:8889"
prometheusremotewrite:
endpoint: http://192.168.11.100:9090/api/v1/write
target_info:
enabled: true
connectors:
spanmetrics:
histogram:
explicit:
buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
dimensions:
- name: http.method
default: GET
- name: http.status_code
dimensions_cache_size: 1000
aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"
metrics_flush_interval: 15s
processors:
batch:
extensions:
health_check:
pprof:
endpoint: :1888
zpages:
endpoint: :55679
service:
extensions: [pprof, zpages, health_check]
pipelines:
traces:
receivers: [jaeger,otlp]
exporters: [spanmetrics, jaeger]
metrics:
receivers: [spanmetrics]
exporters: [prometheusremotewrite]
EOF
另一配置
receivers:
jaeger:
protocols:
thrift_http:
endpoint: :14268
otlp:
protocols:
grpc:
endpoint: :4317
http:
endpoint: :4318
exporters:
jaeger:
endpoint: 10.2.3.8:14250
tls:
insecure: true
prometheus:
endpoint: "10.2.3.8:9464"
resource_to_telemetry_conversion:
enabled: true
enable_open_metrics: true
prometheusremotewrite:
endpoint: http://10.2.3.8:9090/api/v1/write
target_info:
enabled: true
connectors:
spanmetrics:
histogram:
explicit:
buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
dimensions:
- name: http.method
default: GET
- name: http.status_code
dimensions_cache_size: 1000
aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"
processors:
batch:
metricstransform/insert:
transforms:
- include: calls
match_type: strict
action: insert
new_name: calls_total
operations:
- action: update_label
label: span.name
new_label: operation
- include: duration
match_type: strict
action: insert
new_name: latency
operations:
- action: update_label
label: span.name
new_label: operation
extensions:
health_check:
pprof:
endpoint: :1888
zpages:
endpoint: :55679
service:
extensions: [pprof, zpages, health_check]
pipelines:
traces:
receivers: [jaeger,otlp]
processors: [batch]
exporters: [spanmetrics,jaeger]
metrics:
receivers: [spanmetrics]
processors: [metricstransform/insert]
exporters: [prometheusremotewrite]