httpserver 代码
package main
import (
"context"
"fmt"
"github.com/fsnotify/fsnotify"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"io"
"net"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
)
var logLevel string
var viperInstance *viper.Viper
func main() {
httpserverConf := os.Getenv("HTTPSERVER_CONF")
log.Info("configFile from env is " + httpserverConf)
if httpserverConf == "" {
httpserverConf = "/etc/httpserver/httpserver.properties"
}
log.Info("confFile is " + httpserverConf)
viperInstance = viper.New() // viper实例
viperInstance.SetConfigFile(httpserverConf) // 指定配置文件路径
err := viperInstance.ReadInConfig()
if err != nil { // 处理读取配置文件的错误
panic(fmt.Errorf("Fatal error config file: %s \n", err))
}
viperInstance.WatchConfig()
viperInstance.OnConfigChange(func(e fsnotify.Event) {
fmt.Println("Detect config change: %s \n", e.String())
log.Warn("Config file updated.")
viperLoadConf(viperInstance) // 加载配置的方法
})
viperLoadConf(viperInstance)
port := 8080
portstr := ":" + strconv.Itoa(port)
log.Info("httpserver listend port: ", port)
mux := http.NewServeMux()
mux.HandleFunc("/", HttpHandler)
mux.HandleFunc("/healthz", HealthZ)
mux.HandleFunc("/sleep", SleepTest)
server := &http.Server{
Addr: portstr,
Handler: mux,
}
go server.ListenAndServe()
listenSignal(context.Background(), server)
}
func dynamicConfig() {
viperInstance.WatchConfig()
viperInstance.OnConfigChange(func(e fsnotify.Event) {
fmt.Println("Detect config change: %s \n", e.String())
log.Warn("Config file updated.")
viperLoadConf(viperInstance) // 加载配置的方法
})
}
func viperLoadConf(viperInstance *viper.Viper) {
logLevel = viperInstance.GetString("log_level")
level, err := log.ParseLevel(logLevel)
if err != nil {
level = log.GetLevel()
}
log.SetLevel(level)
myconf := viperInstance.GetString("my_conf")
log.Trace(myconf + " in viperLoadConf")
log.Debug(myconf + " in viperLoadConf")
log.Info(myconf + " in viperLoadConf")
log.Warn(myconf + " in viperLoadConf")
log.Error(myconf + " in viperLoadConf")
}
func HealthZ(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
}
func HttpHandler(w http.ResponseWriter, r *http.Request) {
myconf := viperInstance.GetString("my_conf")
log.Trace(myconf)
log.Debug(myconf)
log.Info(myconf)
log.Warn(myconf)
log.Error(myconf)
ip := ClientIP(r)
httpcode := 200
log.Info("reqLog: clientIP : " + ip + " httpcode " + strconv.Itoa(httpcode))
headers := r.Header
log.Info("req headers : %+v", headers)
for k, v := range headers {
for _, val := range v {
w.Header().Set(k, val)
}
}
version := os.Getenv("VERSION")
if version != "" {
w.Header().Set("VERSION", version)
}
w.WriteHeader(httpcode)
io.WriteString(w, "ok")
}
func ClientIP(r *http.Request) string {
xForwardedFor := r.Header.Get("X-Forwarded-For")
ip := strings.TrimSpace(strings.Split(xForwardedFor, ",")[0])
if ip != "" {
return ip
}
ip = strings.TrimSpace(r.Header.Get("X-Real-Ip"))
if ip != "" {
return ip
}
if ip, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)); err == nil {
return ip
}
return ""
}
func SleepTest(w http.ResponseWriter, r *http.Request) {
values := r.URL.Query()
sleeptimeStr := values.Get("sleep")
sleeptime, err := strconv.Atoi(sleeptimeStr)
if err != nil {
sleeptime = 1
}
time.Sleep(time.Duration(sleeptime) * time.Second)
fmt.Fprintln(w, "Hello world, sleep " + strconv.Itoa(sleeptime) + "s")
log.Printf( "Hello world, sleep %+vs", sleeptime)
}
// 优雅终止
func listenSignal(ctx context.Context, httpSrv *http.Server) {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
select {
case <-sigs:
fmt.Println("notify sigs")
httpSrv.Shutdown(ctx)
fmt.Println("http shutdown gracefully")
}
}
Deployment Yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpserver
namespace: mxs
labels:
app: httpserver
spec:
replicas: 2
strategy:
type: RollingUpdate
rollingUpdate:
# maxSurge: 最大激增数, 指更新过程中, 最多可以比replicas预先设定值多出的pod数量, 可以为固定值或百分比(默认25%), 更新过程中最多会有replicas + maxSurge个pod
maxSurge: 2
# maxUnavailable: 最大无效数, 指更新过程中, 最多有几个pod处于无法服务状态, 当maxSurge不为0时, 此栏位也不可为0, 整个更新过程中, 会有maxUnavailable个pod处于Terminating状态
maxUnavailable: 1
# minReadySeconds: 容器内应用的启动时间, pod变为run状态, 会在minReadySeconds后继续更新下一个pod. 如果不设置该属性, pod会在run成功后, 立即更新下一个pod.
minReadySeconds: 15
selector:
matchLabels:
app: httpserver
template:
metadata:
labels:
app: httpserver
spec:
containers:
- name: httpserver
image: tjudream/httpserver:v5
command: [/httpserver]
envFrom:
- configMapRef:
name: httpserver-env-cm
volumeMounts:
- name: config-volume
mountPath: /etc/httpserver/
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 500m
memory: 512Mi
# 优雅启动
livenessProbe:
httpGet:
### this probe will fail with 404 error code
### only httpcode between 200-400 is retreated as success
path: /healthz
port: 8080
initialDelaySeconds: 10
periodSeconds: 5
# 探活
readinessProbe:
httpGet:
### this probe will fail with 404 error code
### only httpcode between 200-400 is retreated as success
path: /healthz
port: 8080
initialDelaySeconds: 30
periodSeconds: 5
successThreshold: 2
volumes:
- name: config-volume
configMap:
# Provide the name of the ConfigMap containing the files you want
# to add to the container
name: httpserver-conf-cm
ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
name: httpserver-env-cm
namespace: mxs
data:
VERSION: v1.0 from cm
HTTPSERVER_CONF: /etc/httpserver/httpserver.properties
---
apiVersion: v1
kind: ConfigMap
metadata:
name: httpserver-conf-cm
namespace: mxs
data:
httpserver.properties: |
log_level: debug
my_conf: myconf_v1
优雅启动
deploy 中的 livenessProbe 提供优雅启动功能，只有当 8080 端口的 http get 请求 url 127.0.0.1/healthz 返回 200 时，pod 才会被 k8s 设置成 ready 状态，然后才会接收外部流量
livenessProbe:
httpGet:
### this probe will fail with 404 error code
### only httpcode between 200-400 is retreated as success
path: /healthz
port: 8080
initialDelaySeconds: 10
periodSeconds: 5
优雅终止
go 代码中的 listenSignal 函数提供优雅终止功能，当 go 的 httpserver 服务接收到 k8s 的终止信号时，会调用 httpserver 的 Shutdown 函数，不再接收新的请求，将现在正在处理中的请求处理完成后会主动退出
// 优雅终止
func listenSignal(ctx context.Context, httpSrv *http.Server) {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
select {
case <-sigs:
fmt.Println("notify sigs")
httpSrv.Shutdown(ctx)
fmt.Println("http shutdown gracefully")
}
}
资源需求和 QoS 保证
k8s 的 QoS 保证：
- pod中所有容器都必须统一设置limits，并且设置参数都一致，如果有一个容器要设置requests，那么所有容器都要设置，并设置参数同limits一致，那么这个pod的QoS就是Guaranteed级别
- Burstable: pod中只要有一个容器的requests和limits的设置不相同，该pod的QoS即为Burstable
- Best-Effort：如果对于全部的resources来说requests与limits均未设置，该pod的QoS即为Best-Effort
3种QoS优先级从有低到高（从左向右）：Best-Effort pods -> Burstable pods -> Guaranteed pods
httpserver 这个容器设置了一样的 requests 和 limit 且只有这一个容器，所以是最高 QoS 保证，即 Guaranteed
探活
deploy 中配置的 readinessProbe 用于探活，一旦 /healthz 这个接口请求不通，或者返回非 200~400 的状态码，则 k8s 就会将这个pod调度为非 ready 状态，流量就不会请求到这个 pod 上了
readinessProbe:
httpGet:
### this probe will fail with 404 error code
### only httpcode between 200-400 is retreated as success
path: /healthz
port: 8080
initialDelaySeconds: 30
periodSeconds: 5
successThreshold: 2
日常运维需求，日志等级
日常日志查看，其中 httpserver-565798b9f9-4rghf 是 pod 的名称，由于 httpserver 将日志打印到控制台中所以可以直接通过 logs 命令查看
kubectl logs -f httpserver-565798b9f9-4rghf
但是在pod出现异常退出时，日志也就销毁了，这种方法就无法查看到日志了
常用的日志解决方案是采用 ELK 方案，搭建 ElasticSearch 集群，采用 filebeat（最初是 logstash，由于内存消耗比较大，所以现在一般用 filebeat 替代）采集日志存储至 ES 中，最后通过 Kibana 查询和展示。
filebeat 可以采用 sidecar 的方式挂载到业务容器中，也可通过 DeamonSet 的方式启动之后进行收集。
配置和代码分离
通过 ConfigMap 配置 httpserver 的配置
envFrom:
- configMapRef:
name: httpserver-env-cm
volumeMounts:
- name: config-volume
mountPath: /etc/httpserver/httpserver.properties
ConfigMap + viper 实现 httpserver 热加载配置
viperInstance.WatchConfig()
viperInstance.OnConfigChange(func(e fsnotify.Event) {
fmt.Println("Detect config change: %s \n", e.String())
log.Warn("Config file updated.")
viperLoadConf(viperInstance) // 加载配置的方法
})
当 ConfigMap 发生变化时，会被 viperInstance.WatchConfig() 监控到，然后会调用 OnConfigChange 函数中的匿名函数，重新加载配置
HPA 动态扩缩容
HPA的全称为Horizontal Pod Autoscaling，它可以根据当前pod资源的使用率（如CPU、磁盘、内存等），进行副本数的动态的扩容与缩容，以便减轻各个pod的压力。
安装 meritcs
wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
在 image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1 上边加入一行 - --kubelet-insecure-tls
image.png
kubectl apply -f components.yaml
创建 hpa，在 cpu 使用率高于 80% 或者内存使用率高于 70% 的时候进行扩容，最少 2 个实例，最多 5 个实例
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: httpserver
namespace: mxs
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: httpserver
minReplicas: 2
maxReplicas: 5
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 80
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 70