Openresty集成prometheus_lua和skywalking_lua笔记

Github文档

skywalking-nginx-lua

nginx-lua-prometheus

Openresty部署-Docker方式

创建目录

mkdir -pv /data/openresty/lua_lib
mkdir -pv /data/openresty/nginx/{logs/vhosts/upstream}

下载对应lua文件到/data/openresty/lua_lib目录, 结构如下

tree
.
├── nginx-lua-prometheus
│   ├── prometheus_keys.lua
│   ├── prometheus.lua
│   ├── prometheus_resty_counter.lua
│   └── prometheus_test.lua
├── nginx-lua-prometheus-0.20220127.tar.gz
├── skywalking-nginx-lua
│   ├── resty
│   │   ├── http_headers.lua
│   │   ├── http.lua
│   │   └── jit-uuid.lua
│   └── skywalking
│       ├── client.lua
│       ├── constants.lua
│       ├── correlation_context.lua
│       ├── correlation_context_test.lua
│       ├── dependencies
│       │   └── base64.lua
│       ├── management.lua
│       ├── segment.lua
│       ├── segment_ref.lua
│       ├── segment_ref_test.lua
│       ├── span_layer.lua
│       ├── span.lua
│       ├── span_test.lua
│       ├── tracer.lua
│       ├── tracing_context.lua
│       ├── tracing_context_test.lua
│       ├── util.lua
│       └── util_test.lua
└── skywalking-nginx-lua-0.6.0.tar.gz

编辑nginx配置文件

nginx.conf

cat > /data/openresty/nginx/nginx.conf << 'EOF'
user  nobody;
worker_processes auto;

# Enables the use of JIT for regular expressions to speed-up their processing.
pcre_jit on;

error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  4096;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    lua_package_path "/usr/local/openresty/lualib/skywalking-nginx-lua/?.lua;/usr/local/openresty/lualib/nginx-lua-prometheus/?.lua;;";

    lua_shared_dict tracing_buffer 100M;
    lua_shared_dict prometheus_metrics 100M;

    init_worker_by_lua_block {
        local metadata_buffer = ngx.shared.tracing_buffer
        metadata_buffer:set('serviceName', 'Openresty-test')
        metadata_buffer:set('serviceInstanceName', 'ccms-credit-front01-test')
        metadata_buffer:set('includeHostInEntrySpan', false)
        require("skywalking.util").set_randomseed()
        require("skywalking.client"):startBackendTimer("http://3.1.101.39:12800")
        skywalking_tracer = require("skywalking.tracer");

        prometheus = require("prometheus").init("prometheus_metrics")
        metric_requests = prometheus:counter("nginx_http_requests_total", "Number of HTTP requests", {"server_name", "status"})
        metric_latency = prometheus:histogram("nginx_http_request_duration_seconds", "HTTP request latency", {"server_name"})
        metric_connections = prometheus:gauge("nginx_http_connections", "Number of HTTP connections", {"state"})
        metric_requests_uri = prometheus:counter("nginx_http_requests_uri_total", "Number of HTTP requests_uri", {"server_name", "port", "uri", "status", "method"})
    }

    log_by_lua_block {
        metric_requests:inc(1, {ngx.var.server_name, ngx.var.status})
        metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name})
        metric_requests_uri:inc(1, {ngx.var.server_name,ngx.var.server_port,ngx.var.document_uri, ngx.var.status, ngx.var.request_method})
    }


    # Enables or disables the use of underscores in client request header fields.
    # When the use of underscores is disabled, request header fields whose names contain underscores are marked as invalid and become subject to the ignore_invalid_headers directive.
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format json
      '{"@timestamp":"$time_iso8601",'
      '"host":"$hostname",'
      '"server_ip":"$server_addr",'
      '"client_ip":"$remote_addr",'
      '"xff":"$http_x_forwarded_for",'
      '"domain":"$host",'
      '"url":"$uri",'
      '"referer":"$http_referer",'
      '"args":"$args",'
      '"upstreamtime":"$upstream_response_time",'
      '"responsetime":"$request_time",'
      '"request_method":"$request_method",'
      '"status":"$status",'
      '"size":"$body_bytes_sent",'
      #'"request_body":"$request_body",'
      '"request_length":"$request_length",'
      '"protocol":"$server_protocol",'
      '"upstreamhost":"$upstream_addr",'
      '"file_dir":"$request_filename",'
      '"http_user_agent":"$http_user_agent"'
    '}';

    access_log logs/access.log json;

    # See Move default writable paths to a dedicated directory (#119)
    # https://github.com/openresty/docker-openresty/issues/119
    client_body_temp_path /var/run/openresty/nginx-client-body;
    proxy_temp_path       /var/run/openresty/nginx-proxy;
    fastcgi_temp_path     /var/run/openresty/nginx-fastcgi;
    uwsgi_temp_path       /var/run/openresty/nginx-uwsgi;
    scgi_temp_path        /var/run/openresty/nginx-scgi;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;
    fastcgi_buffer_size 128k;
    fastcgi_buffers 8 128k;
    fastcgi_busy_buffers_size 128k;
    fastcgi_temp_file_write_size 128k;

    client_header_buffer_size 128k;
    client_body_buffer_size 1m;

    underscores_in_headers on;
    include /etc/nginx/conf.d/*.conf;

    # Don't reveal OpenResty version to clients.
    # server_tokens off;
}
include /etc/nginx/upstream/*.conf;
EOF

server-skywalking.conf

cat > /data/openresty/nginx/vhosts/credit.conf << 'EOF'
server {
    listen       8901;
    listen  [::]:8901;
    server_name  localhost;

    location / {
        rewrite_by_lua_block {
            skywalking_tracer:start("credit-ROOT")
        }
        body_filter_by_lua_block {
            if ngx.arg[2] then
                skywalking_tracer:finish()
            end
        }
        log_by_lua_block {
            skywalking_tracer:prepareForReport()
        }
        root   /opt/ccms-auto-deploy/front-bank-credit;
        index  index.html index.htm;
    }

    location /login/ {
        rewrite_by_lua_block {
            skywalking_tracer:start("credit-login")
        }
        body_filter_by_lua_block {
            if ngx.arg[2] then
                skywalking_tracer:finish()
            end
        }
        log_by_lua_block {
            skywalking_tracer:prepareForReport()
        }

        proxy_pass http://3.1.101.55:11204/;
        proxy_redirect off;
        proxy_set_header HOST $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    }
    access_log  logs/ccms-credit-access.log json;
}
EOF

server-prometheus.conf

cat > /data/openresty/nginx/vhosts/prometheus.conf << 'EOF'
server {
  listen 9145;
  server_name  localhost;

  location /metrics {
    content_by_lua_block {
      metric_connections:set(ngx.var.connections_reading, {"reading"})
      metric_connections:set(ngx.var.connections_waiting, {"waiting"})
      metric_connections:set(ngx.var.connections_writing, {"writing"})
      prometheus:collect()
    }
  }
}
EOF

编辑docker-compose容器编排文件

cat > /data/docker-compose/openresty/docker-compose.yml << 'EOF'
version: "3"
services:
  openresty:
    container_name: openresty
    hostname: openresty-01
    image: openresty/openresty:1.19.9.1-centos7
    network_mode: host
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime
      - /data/openresty/lua_lib/nginx-lua-prometheus:/usr/local/openresty/lualib/nginx-lua-prometheus
      - /data/openresty/lua_lib/skywalking-nginx-lua:/usr/local/openresty/lualib/skywalking-nginx-lua
      - /data/openresty/nginx/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf
      - /data/openresty/nginx/upstream:/etc/nginx/upstream
      - /data/openresty/nginx/vhosts:/etc/nginx/conf.d
      - /data/openresty/nginx/logs:/usr/local/openresty/nginx/logs
      - /opt/ccms-auto-deploy:/opt/ccms-auto-deploy
    deploy:
      resources:
        limits:
           memory: 4G
EOF

启动

cd /data/docker-compose/openresty; docker-compose up -d