准备环境
go,docker,有2个worker node的k8s集群(对两个node分别打label,tenantID=1,tenantID=2)
准备代码和manifest
创建目录myscheduler,进入目录
mkdir myscheduler
cd myscheduler
创建main.go,内容如下
package main
import (
"context"
"fmt"
"math/rand"
"os"
"sync"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
_ "k8s.io/component-base/logs/json/register"
"k8s.io/kubernetes/cmd/kube-scheduler/app"
"k8s.io/kubernetes/pkg/scheduler/framework"
)
const (
name = "myscheduler"
)
func main() {
rand.Seed(time.Now().UnixNano())
command := app.NewSchedulerCommand(
app.WithPlugin(name, New),
)
if err := command.Execute(); err != nil {
os.Exit(1)
}
}
var _ framework.FilterPlugin = &Scheduler{}
func New(_ runtime.Object, handle framework.Handle) (framework.Plugin, error) {
scheduler := &Scheduler{}
return scheduler, nil
}
type Scheduler struct {
sync.Mutex
}
func (s *Scheduler) Name() string {
return name
}
const (
tenantIDLabel = "tenantID"
)
func (s *Scheduler) Filter(ctx context.Context, state *framework.CycleState, pod *corev1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
node := nodeInfo.Node()
if node == nil {
return framework.NewStatus(framework.Error, "node not found")
}
tenantID := ""
if pod.Labels != nil {
tenantID = pod.Labels[tenantIDLabel]
}
if node.Labels != nil && node.Labels[tenantIDLabel] == tenantID {
return framework.NewStatus(framework.Success, "success")
}
return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("node:%s not belong to tenantID:%s", node.Name, tenantID))
}
创建go.mod,内容如下
module myscheduler
go 1.18
require (
k8s.io/api v0.23.3
k8s.io/apimachinery v0.23.3
k8s.io/component-base v0.23.3
k8s.io/kubernetes v1.23.3
)
require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/emicklei/go-restful v2.9.5+incompatible // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/go-logr/zapr v1.2.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.5 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/gomega v1.17.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/runc v1.0.2 // indirect
github.com/opencontainers/selinux v1.8.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.11.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.28.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/spf13/cobra v1.2.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/v3 v3.5.0 // indirect
go.opentelemetry.io/contrib v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
go.opentelemetry.io/otel/metric v0.20.0 // indirect
go.opentelemetry.io/otel/sdk v0.20.0 // indirect
go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
go.opentelemetry.io/otel/trace v0.20.0 // indirect
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/goleak v1.1.12 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
google.golang.org/grpc v1.40.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/apiserver v0.23.3 // indirect
k8s.io/client-go v0.23.3 // indirect
k8s.io/cloud-provider v0.23.3 // indirect
k8s.io/component-helpers v0.23.3 // indirect
k8s.io/csi-translation-lib v0.23.3 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/kube-scheduler v0.23.3 // indirect
k8s.io/mount-utils v0.23.3 // indirect
k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27 // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
replace (
k8s.io/api => k8s.io/api v0.23.3
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.3
k8s.io/apimachinery => k8s.io/apimachinery v0.23.3
k8s.io/apiserver => k8s.io/apiserver v0.23.3
k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.3
k8s.io/client-go => k8s.io/client-go v0.23.3
k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.3
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.3
k8s.io/code-generator => k8s.io/code-generator v0.23.3
k8s.io/component-base => k8s.io/component-base v0.23.3
k8s.io/component-helpers => k8s.io/component-helpers v0.23.3
k8s.io/controller-manager => k8s.io/controller-manager v0.23.3
k8s.io/cri-api => k8s.io/cri-api v0.23.3
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.3
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.3
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.3
k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.3
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.3
k8s.io/kubectl => k8s.io/kubectl v0.23.3
k8s.io/kubelet => k8s.io/kubelet v0.23.3
k8s.io/kubernetes => k8s.io/kubernetes v1.23.3
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.3
k8s.io/metrics => k8s.io/metrics v0.23.3
k8s.io/mount-utils => k8s.io/mount-utils v0.23.3
k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.3
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.3
)
执行
go mod tidy
创建Dockerfile,内容如下
FROM golang:alpine as builder
ENV GO111MODULE=on GOPROXY=https://goproxy.cn,direct
ARG CONF=dev
WORKDIR /go/app
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY main.go main.go
RUN CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -o app .
FROM alpine:latest as prod
ARG CONF=dev
RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && apk --no-cache add ca-certificates && apk add tzdata && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo "Asia/Shanghai" > /etc/timezone
WORKDIR /app/
COPY --from=0 /go/app/app .
CMD ["./app"]
创建configmap.yaml,内容如下
apiVersion: v1
kind: ConfigMap
metadata:
name: myscheduler
namespace: mynamespace
data:
conf.yaml: |
apiVersion: kubescheduler.config.k8s.io/v1beta3
kind: KubeSchedulerConfiguration
leaderElection:
leaderElect: true
leaseDuration: 15s
renewDeadline: 10s
retryPeriod: 2s
resourceLock: leases
resourceName: myscheduler
resourceNamespace: mynamespace
profiles:
- schedulerName: myscheduler
plugins:
filter:
enabled:
- name: "myscheduler"
创建deployment.yaml,内容如下
apiVersion: apps/v1
kind: Deployment
metadata:
name: myscheduler
namespace: mynamespace
spec:
replicas: 1
selector:
matchLabels:
app: myscheduler
template:
metadata:
labels:
app: myscheduler
spec:
serviceAccount: myscheduler
priorityClassName: system-cluster-critical
volumes:
- name: conf
configMap:
name: myscheduler
containers:
- name: app
image: myscheduler:0.1
imagePullPolicy: IfNotPresent
command:
- ./app
- --config=/scheduler/conf.yaml
- --v=3
- --logging-format=json
volumeMounts:
- name: conf
mountPath: /scheduler
创建rbac.yaml,内容如下
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: myscheduler
namespace: mynamespace
rules:
- apiGroups:
- events.k8s.io
resources:
- events
verbs:
- create
- apiGroups:
- storage.k8s.io
resources:
- csistoragecapacities
- csidrivers
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
- nodes
- pods
- persistentvolumes
- persistentvolumeclaims
- services
- replicationcontrollers
- configmaps
verbs:
- get
- list
- watch
- update
- patch
- create
- apiGroups:
- ""
resources:
- events
- pods/status
- pods/binding
verbs:
- get
- list
- watch
- create
- update
- patch
- apiGroups:
- "apps"
resources:
- replicasets
- statefulsets
verbs:
- get
- list
- watch
- apiGroups:
- "coordination.k8s.io"
resources:
- leases
verbs:
- get
- list
- watch
- create
- update
- patch
- apiGroups:
- "policy"
resources:
- poddisruptionbudgets
verbs:
- get
- list
- watch
- apiGroups:
- "storage.k8s.io"
resources:
- storageclasses
- csinodes
verbs:
- get
- list
- watch
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: myscheduler
namespace: mynamespace
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: myscheduler
namespace: mynamespace
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: myscheduler
subjects:
- kind: ServiceAccount
name: myscheduler
namespace: mynamespace
构建镜像
docker build -t myscheduler:0.1 .
创建k8s资源
kubectl create ns mynamespace
kubectl apply -f configmap.yaml
kubectl apply -f rbac.yaml
kubectl apply -f deplopment.yaml
实验
apiVersion: v1
kind: Pod
metadata:
name: schedulerdemo
namespace: mynamespace
labels:
tenantID: "1"
spec:
schedulerName: "myscheduler"
containers:
- command:
- 'sleep 5'
image: busybox
name: busybox
查看pod对应的node
kubectl get pod -n mynamespace schedulerdemo