Kubernetes API Server源码学习（四）：Admission机制的实现、HttpReq的处理过程、Authentication与Authorization
本文基于Kubernetes v1.22.4版本进行源码学习
12、Admission机制的实现
1）、Admission简介
Admission Controller在HTTP请求经过登录和鉴权之后并且在Request真正被处理并且存储到Etcd之前去执行，可以修改请求对象（Mutation）或校验请求（Validation）
一个Request进入API Server后会经过如下的处理过程：
- Authentication Authorization：登录和鉴权，校验Request发送者是否合法
- Decode & Conversion：Request以JSON格式发过来，转换为Go结构体类型，将外部版本转换为API Server内部APIObject版本
- Admission - Mutation：拿到Request对内容进行调整
- Admission - Mutation Webhook：通过Webhook调用Kubernetes使用者扩展的Mutation逻辑
- Admission - Validation：拿到Request对内容进行校验
- Admission - Validation Webhook：通过Webhook调用Kubernetes使用者扩展的Validation逻辑
- ETCD：将Request落实到ETCD中
2）、内建的Admission Plugin
在启动API Server时，有些Admission是默认启用的，使用者可以通过启动参数来启动或禁用指定Admission，但无法影响一个Admission内部的逻辑
// pkg/kubeapiserver/options/plugins.go
var AllOrderedPlugins = []string{
admit.PluginName, // AlwaysAdmit
autoprovision.PluginName, // NamespaceAutoProvision
lifecycle.PluginName, // NamespaceLifecycle
exists.PluginName, // NamespaceExists
scdeny.PluginName, // SecurityContextDeny
antiaffinity.PluginName, // LimitPodHardAntiAffinityTopology
limitranger.PluginName, // LimitRanger
serviceaccount.PluginName, // ServiceAccount
noderestriction.PluginName, // NodeRestriction
nodetaint.PluginName, // TaintNodesByCondition
alwayspullimages.PluginName, // AlwaysPullImages
imagepolicy.PluginName, // ImagePolicyWebhook
podsecuritypolicy.PluginName, // PodSecurityPolicy
podsecurity.PluginName, // PodSecurity
podnodeselector.PluginName, // PodNodeSelector
podpriority.PluginName, // Priority
defaulttolerationseconds.PluginName, // DefaultTolerationSeconds
podtolerationrestriction.PluginName, // PodTolerationRestriction
eventratelimit.PluginName, // EventRateLimit
extendedresourcetoleration.PluginName, // ExtendedResourceToleration
label.PluginName, // PersistentVolumeLabel
setdefault.PluginName, // DefaultStorageClass
storageobjectinuseprotection.PluginName, // StorageObjectInUseProtection
gc.PluginName, // OwnerReferencesPermissionEnforcement
resize.PluginName, // PersistentVolumeClaimResize
runtimeclass.PluginName, // RuntimeClass
certapproval.PluginName, // CertificateApproval
certsigning.PluginName, // CertificateSigning
certsubjectrestriction.PluginName, // CertificateSubjectRestriction
defaultingressclass.PluginName, // DefaultIngressClass
denyserviceexternalips.PluginName, // DenyServiceExternalIPs
// new admission plugins should generally be inserted above here
// webhook, resourcequota, and deny plugins must go at the end
mutatingwebhook.PluginName, // MutatingAdmissionWebhook
validatingwebhook.PluginName, // ValidatingAdmissionWebhook
resourcequota.PluginName, // ResourceQuota
deny.PluginName, // AlwaysDeny
}
其中有三个特殊的Admission Plugin：ImagePolicyWebhook、MutatingAdmissionWebhook、ValidatingAdmissionWebhook，它们会根据设置去调用使用者自己写的Web服务，传入请求的目标Object，让该服务判断是否需要拒绝、允许或进行修改。这是Kubernetes所提供的标准扩展方式之一
3）、Admission Plugin的装配
1）加载Admission Option
函数之间调用逻辑如下：
最核心的是RegisterAllAdmissionPlugins()方法，代码如下：
// pkg/kubeapiserver/options/plugins.go
func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
// 把所有的Admission Plugin都拿出来,通过传入的参数plugins带出去,最后被交到option的Admission属性上
admit.Register(plugins) // DEPRECATED as no real meaning
alwayspullimages.Register(plugins)
antiaffinity.Register(plugins)
defaulttolerationseconds.Register(plugins)
defaultingressclass.Register(plugins)
denyserviceexternalips.Register(plugins)
deny.Register(plugins) // DEPRECATED as no real meaning
eventratelimit.Register(plugins)
extendedresourcetoleration.Register(plugins)
gc.Register(plugins)
imagepolicy.Register(plugins)
limitranger.Register(plugins)
autoprovision.Register(plugins)
exists.Register(plugins)
noderestriction.Register(plugins)
nodetaint.Register(plugins)
label.Register(plugins) // DEPRECATED, future PVs should not rely on labels for zone topology
podnodeselector.Register(plugins)
podtolerationrestriction.Register(plugins)
runtimeclass.Register(plugins)
resourcequota.Register(plugins)
podsecurity.Register(plugins) // before PodSecurityPolicy so audit/warn get exercised even if PodSecurityPolicy denies
podsecuritypolicy.Register(plugins)
podpriority.Register(plugins)
scdeny.Register(plugins)
serviceaccount.Register(plugins)
setdefault.Register(plugins)
resize.Register(plugins)
storageobjectinuseprotection.Register(plugins)
certapproval.Register(plugins)
certsigning.Register(plugins)
certsubjectrestriction.Register(plugins)
}
2）根据Admission Option配置APIServer Config
函数之间调用逻辑如下：
最核心的是ApplyTo()方法，代码如下：
// vendor/k8s.io/apiserver/pkg/server/options/admission.go
func (a *AdmissionOptions) ApplyTo(
c *server.Config,
informers informers.SharedInformerFactory,
kubeAPIServerClientConfig *rest.Config,
features featuregate.FeatureGate,
pluginInitializers ...admission.PluginInitializer,
) error {
if a == nil {
return nil
}
// Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
if informers == nil {
return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
}
pluginNames := a.enabledPluginNames()
pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
if err != nil {
return fmt.Errorf("failed to read plugin config: %v", err)
}
clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
if err != nil {
return err
}
genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, features)
initializersChain := admission.PluginInitializers{}
pluginInitializers = append(pluginInitializers, genericInitializer)
initializersChain = append(initializersChain, pluginInitializers...)
// 启用的Admission Plugin生成admissionChain
admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
if err != nil {
return err
}
// 将admissionChain被放入server.Config的AdmissionControl属性
c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain)
return nil
}
3）Admission Plugin注入到Request Handler中
API Resource的装载流程中和Admission相关的逻辑如下：
以PUT请求为例，registerResourceHandlers()方法中调用逻辑如下：
vendor/k8s.io/apiserver/pkg/endpoints/handlers包下每一个文件对应一个http verb，负责构建对该verb的handler
UpdateResource()方法代码如下：
// vendor/k8s.io/apiserver/pkg/endpoints/handlers/update.go
// admit为其输入参数
func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interface) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
// For performance tracking purposes.
trace := utiltrace.New("Update", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
namespace, name, err := scope.Namer.Name(req)
if err != nil {
scope.err(err, w, req)
return
}
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
if err != nil {
scope.err(err, w, req)
return
}
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
}
options := &metav1.UpdateOptions{}
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil {
err = errors.NewBadRequest(err.Error())
scope.err(err, w, req)
return
}
if errs := validation.ValidateUpdateOptions(options); len(errs) > 0 {
err := errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "UpdateOptions"}, "", errs)
scope.err(err, w, req)
return
}
options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("UpdateOptions"))
s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer)
if err != nil {
scope.err(err, w, req)
return
}
defaultGVK := scope.Kind
original := r.New()
trace.Step("About to convert to expected version")
decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)
obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
if err != nil {
err = transformDecodeError(scope.Typer, err, original, gvk, body)
scope.err(err, w, req)
return
}
objGV := gvk.GroupVersion()
if !scope.AcceptsGroupVersion(objGV) {
err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%s)", objGV, defaultGVK.GroupVersion()))
scope.err(err, w, req)
return
}
trace.Step("Conversion done")
ae := request.AuditEventFrom(ctx)
audit.LogRequestObject(ae, obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)
admit = admission.WithAudit(admit, ae)
if err := checkName(obj, name, namespace, scope.Namer); err != nil {
scope.err(err, w, req)
return
}
userInfo, _ := request.UserFrom(ctx)
transformers := []rest.TransformFunc{}
// allows skipping managedFields update if the resulting object is too big
shouldUpdateManagedFields := true
if scope.FieldManager != nil {
admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
transformers = append(transformers, func(_ context.Context, newObj, liveObj runtime.Object) (runtime.Object, error) {
if shouldUpdateManagedFields {
return scope.FieldManager.UpdateNoErrors(liveObj, newObj, managerOrUserAgent(options.FieldManager, req.UserAgent())), nil
}
return newObj, nil
})
}
// 1)用mutation admission来构造Transformer,用于更改被更新的目标Object
if mutatingAdmission, ok := admit.(admission.MutationInterface); ok {
transformers = append(transformers, func(ctx context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) {
isNotZeroObject, err := hasUID(oldObj)
if err != nil {
return nil, fmt.Errorf("unexpected error when extracting UID from oldObj: %v", err.Error())
} else if !isNotZeroObject {
if mutatingAdmission.Handles(admission.Create) {
return newObj, mutatingAdmission.Admit(ctx, admission.NewAttributesRecord(newObj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, updateToCreateOptions(options), dryrun.IsDryRun(options.DryRun), userInfo), scope)
}
} else {
if mutatingAdmission.Handles(admission.Update) {
return newObj, mutatingAdmission.Admit(ctx, admission.NewAttributesRecord(newObj, oldObj, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Update, options, dryrun.IsDryRun(options.DryRun), userInfo), scope)
}
}
return newObj, nil
})
transformers = append(transformers, func(ctx context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) {
// Dedup owner references again after mutating admission happens
dedupOwnerReferencesAndAddWarning(newObj, req.Context(), true)
return newObj, nil
})
}
createAuthorizerAttributes := authorizer.AttributesRecord{
User: userInfo,
ResourceRequest: true,
Path: req.URL.Path,
Verb: "create",
APIGroup: scope.Resource.Group,
APIVersion: scope.Resource.Version,
Resource: scope.Resource.Resource,
Subresource: scope.Subresource,
Namespace: namespace,
Name: name,
}
trace.Step("About to store object in database")
wasCreated := false
// 2)调用r.Update()方法
requestFunc := func() (runtime.Object, error) {
obj, created, err := r.Update(
ctx,
name,
rest.DefaultUpdatedObjectInfo(obj, transformers...),
withAuthorization(rest.AdmissionToValidateObjectFunc(
admit,
admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, updateToCreateOptions(options), dryrun.IsDryRun(options.DryRun), userInfo), scope),
scope.Authorizer, createAuthorizerAttributes),
rest.AdmissionToValidateObjectUpdateFunc(
admit,
admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Update, options, dryrun.IsDryRun(options.DryRun), userInfo), scope),
false,
options,
)
wasCreated = created
return obj, err
}
// Dedup owner references before updating managed fields
dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
result, err := requestFunc()
// If the object wasn't committed to storage because it's serialized size was too large,
// it is safe to remove managedFields (which can be large) and try again.
if isTooLargeError(err) && scope.FieldManager != nil {
if accessor, accessorErr := meta.Accessor(obj); accessorErr == nil {
accessor.SetManagedFields(nil)
shouldUpdateManagedFields = false
result, err = requestFunc()
}
}
return result, err
})
if err != nil {
scope.err(err, w, req)
return
}
trace.Step("Object stored in database")
status := http.StatusOK
if wasCreated {
status = http.StatusCreated
}
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
}
}
UpdateResource()方法中使用Mutation Admission来构造Transformer，用于更改被更新的目标Object，然后调用r.Update()方法，该方法实现如下：
// vendor/k8s.io/apiserver/pkg/registry/generic/registry/store.go
func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
key, err := e.KeyFunc(ctx, name)
if err != nil {
return nil, false, err
}
var (
creatingObj runtime.Object
creating = false
)
qualifiedResource := e.qualifiedResourceFromContext(ctx)
storagePreconditions := &storage.Preconditions{}
if preconditions := objInfo.Preconditions(); preconditions != nil {
storagePreconditions.UID = preconditions.UID
storagePreconditions.ResourceVersion = preconditions.ResourceVersion
}
out := e.NewFunc()
// deleteObj is only used in case a deletion is carried out
var deleteObj runtime.Object
err = e.Storage.GuaranteedUpdate(ctx, key, out, true, storagePreconditions, func(existing runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) {
existingResourceVersion, err := e.Storage.Versioner().ObjectResourceVersion(existing)
if err != nil {
return nil, nil, err
}
if existingResourceVersion == 0 {
if !e.UpdateStrategy.AllowCreateOnUpdate() && !forceAllowCreate {
return nil, nil, apierrors.NewNotFound(qualifiedResource, name)
}
}
// 1)调用transformers,完成mutation Admission
// Given the existing object, get the new object
obj, err := objInfo.UpdatedObject(ctx, existing)
if err != nil {
return nil, nil, err
}
// If AllowUnconditionalUpdate() is true and the object specified by
// the user does not have a resource version, then we populate it with
// the latest version. Else, we check that the version specified by
// the user matches the version of latest storage object.
newResourceVersion, err := e.Storage.Versioner().ObjectResourceVersion(obj)
if err != nil {
return nil, nil, err
}
doUnconditionalUpdate := newResourceVersion == 0 && e.UpdateStrategy.AllowUnconditionalUpdate()
if existingResourceVersion == 0 {
var finishCreate FinishFunc = finishNothing
if e.BeginCreate != nil {
fn, err := e.BeginCreate(ctx, obj, newCreateOptionsFromUpdateOptions(options))
if err != nil {
return nil, nil, err
}
finishCreate = fn
defer func() {
finishCreate(ctx, false)
}()
}
creating = true
creatingObj = obj
if err := rest.BeforeCreate(e.CreateStrategy, ctx, obj); err != nil {
return nil, nil, err
}
// at this point we have a fully formed object. It is time to call the validators that the apiserver
// handling chain wants to enforce.
if createValidation != nil {
if err := createValidation(ctx, obj.DeepCopyObject()); err != nil {
return nil, nil, err
}
}
ttl, err := e.calculateTTL(obj, 0, false)
if err != nil {
return nil, nil, err
}
// The operation has succeeded. Call the finish function if there is one,
// and then make sure the defer doesn't call it again.
fn := finishCreate
finishCreate = finishNothing
fn(ctx, true)
return obj, &ttl, nil
}
creating = false
creatingObj = nil
if doUnconditionalUpdate {
// Update the object's resource version to match the latest
// storage object's resource version.
err = e.Storage.Versioner().UpdateObject(obj, res.ResourceVersion)
if err != nil {
return nil, nil, err
}
} else {
// Check if the object's resource version matches the latest
// resource version.
if newResourceVersion == 0 {
// TODO: The Invalid error should have a field for Resource.
// After that field is added, we should fill the Resource and
// leave the Kind field empty. See the discussion in #18526.
qualifiedKind := schema.GroupKind{Group: qualifiedResource.Group, Kind: qualifiedResource.Resource}
fieldErrList := field.ErrorList{field.Invalid(field.NewPath("metadata").Child("resourceVersion"), newResourceVersion, "must be specified for an update")}
return nil, nil, apierrors.NewInvalid(qualifiedKind, name, fieldErrList)
}
if newResourceVersion != existingResourceVersion {
return nil, nil, apierrors.NewConflict(qualifiedResource, name, fmt.Errorf(OptimisticLockErrorMsg))
}
}
var finishUpdate FinishFunc = finishNothing
if e.BeginUpdate != nil {
fn, err := e.BeginUpdate(ctx, obj, existing, options)
if err != nil {
return nil, nil, err
}
finishUpdate = fn
defer func() {
finishUpdate(ctx, false)
}()
}
if err := rest.BeforeUpdate(e.UpdateStrategy, ctx, obj, existing); err != nil {
return nil, nil, err
}
// 2)使用validation Admission
// at this point we have a fully formed object. It is time to call the validators that the apiserver
// handling chain wants to enforce.
if updateValidation != nil {
if err := updateValidation(ctx, obj.DeepCopyObject(), existing.DeepCopyObject()); err != nil {
return nil, nil, err
}
}
// Check the default delete-during-update conditions, and store-specific conditions if provided
if ShouldDeleteDuringUpdate(ctx, key, obj, existing) &&
(e.ShouldDeleteDuringUpdate == nil || e.ShouldDeleteDuringUpdate(ctx, key, obj, existing)) {
deleteObj = obj
return nil, nil, errEmptiedFinalizers
}
ttl, err := e.calculateTTL(obj, res.TTL, true)
if err != nil {
return nil, nil, err
}
// The operation has succeeded. Call the finish function if there is one,
// and then make sure the defer doesn't call it again.
fn := finishUpdate
finishUpdate = finishNothing
fn(ctx, true)
if int64(ttl) != res.TTL {
return obj, &ttl, nil
}
return obj, nil, nil
}, dryrun.IsDryRun(options.DryRun), nil)
if err != nil {
// delete the object
if err == errEmptiedFinalizers {
return e.deleteWithoutFinalizers(ctx, name, key, deleteObj, storagePreconditions, newDeleteOptionsFromUpdateOptions(options))
}
if creating {
err = storeerr.InterpretCreateError(err, qualifiedResource, name)
err = rest.CheckGeneratedNameError(e.CreateStrategy, err, creatingObj)
} else {
err = storeerr.InterpretUpdateError(err, qualifiedResource, name)
}
return nil, false, err
}
if creating {
if e.AfterCreate != nil {
e.AfterCreate(out, newCreateOptionsFromUpdateOptions(options))
}
} else {
if e.AfterUpdate != nil {
e.AfterUpdate(out, options)
}
}
if e.Decorator != nil {
e.Decorator(out)
}
return out, creating, nil
}
在Storage Object的Update方法前，先调用Transformers，完成Mutation Admission，然后使用Validation Admission，如果Validation出错直接返回
13、HttpReq的处理过程和Default Filters
1）、RequestHandler的构建
一个HTTP请求会由一个HttpHandler对象来处理，该对象具有ServeHTTP()方法。通过装饰器模式，在一个Handler外围不断包裹针对不同方面的处理逻辑，从而形成请求响应的全部流程
在GenericAPIServer的装配过程核心方法New()方法如下：
// vendor/k8s.io/apiserver/pkg/server/config.go
func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
if c.Serializer == nil {
return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
}
if c.LoopbackClientConfig == nil {
return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
}
if c.EquivalentResourceRegistry == nil {
return nil, fmt.Errorf("Genericapiserver.New() called with config.EquivalentResourceRegistry == nil")
}
// 构建handlerChain
// config.BuildHandlerChainFunc的实现为DefaultBuildHandlerChain方法
handlerChainBuilder := func(handler http.Handler) http.Handler {
return c.BuildHandlerChainFunc(handler, c.Config)
}
// 构建NewAPIServerHandler
apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
// 实例化GenericAPIServer
s := &GenericAPIServer{
discoveryAddresses: c.DiscoveryAddresses,
LoopbackClientConfig: c.LoopbackClientConfig,
legacyAPIGroupPrefixes: c.LegacyAPIGroupPrefixes,
admissionControl: c.AdmissionControl,
Serializer: c.Serializer,
AuditBackend: c.AuditBackend,
Authorizer: c.Authorization.Authorizer,
delegationTarget: delegationTarget,
EquivalentResourceRegistry: c.EquivalentResourceRegistry,
HandlerChainWaitGroup: c.HandlerChainWaitGroup,
minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second,
ShutdownTimeout: c.RequestTimeout,
ShutdownDelayDuration: c.ShutdownDelayDuration,
SecureServingInfo: c.SecureServing,
ExternalAddress: c.ExternalAddress,
// 构建了http request的路由,连接了url和响应函数;同时包含了一个request需要经过的预处理函数
Handler: apiServerHandler,
listedPathProvider: apiServerHandler,
openAPIConfig: c.OpenAPIConfig,
skipOpenAPIInstallation: c.SkipOpenAPIInstallation,
// 这些hook集合在New方法接下来的代码中填充,包含自己定义的和delegationTarget上的
postStartHooks: map[string]postStartHookEntry{},
preShutdownHooks: map[string]preShutdownHookEntry{},
disabledPostStartHooks: c.DisabledPostStartHooks,
healthzChecks: c.HealthzChecks,
livezChecks: c.LivezChecks,
readyzChecks: c.ReadyzChecks,
livezGracePeriod: c.LivezGracePeriod,
DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
maxRequestBodyBytes: c.MaxRequestBodyBytes,
livezClock: clock.RealClock{},
lifecycleSignals: c.lifecycleSignals,
APIServerID: c.APIServerID,
StorageVersionManager: c.StorageVersionManager,
Version: c.Version,
}
for {
if c.JSONPatchMaxCopyBytes <= 0 {
break
}
existing := atomic.LoadInt64(&jsonpatch.AccumulatedCopySizeLimit)
if existing > 0 && existing < c.JSONPatchMaxCopyBytes {
break
}
if atomic.CompareAndSwapInt64(&jsonpatch.AccumulatedCopySizeLimit, existing, c.JSONPatchMaxCopyBytes) {
break
}
}
// 处理钩子hook操作
// first add poststarthooks from delegated targets
for k, v := range delegationTarget.PostStartHooks() {
s.postStartHooks[k] = v
}
for k, v := range delegationTarget.PreShutdownHooks() {
s.preShutdownHooks[k] = v
}
// add poststarthooks that were preconfigured. Using the add method will give us an error if the same name has already been registered.
for name, preconfiguredPostStartHook := range c.PostStartHooks {
if err := s.AddPostStartHook(name, preconfiguredPostStartHook.hook); err != nil {
return nil, err
}
}
genericApiServerHookName := "generic-apiserver-start-informers"
if c.SharedInformerFactory != nil {
if !s.isPostStartHookRegistered(genericApiServerHookName) {
err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
c.SharedInformerFactory.Start(context.StopCh)
return nil
})
if err != nil {
return nil, err
}
}
// TODO: Once we get rid of /healthz consider changing this to post-start-hook.
err := s.AddReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory))
if err != nil {
return nil, err
}
}
const priorityAndFairnessConfigConsumerHookName = "priority-and-fairness-config-consumer"
if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) {
} else if c.FlowControl != nil {
err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(context PostStartHookContext) error {
go c.FlowControl.MaintainObservations(context.StopCh)
go c.FlowControl.Run(context.StopCh)
return nil
})
if err != nil {
return nil, err
}
// TODO(yue9944882): plumb pre-shutdown-hook for request-management system?
} else {
klog.V(3).Infof("Not requested to run hook %s", priorityAndFairnessConfigConsumerHookName)
}
// Add PostStartHooks for maintaining the watermarks for the Priority-and-Fairness and the Max-in-Flight filters.
if c.FlowControl != nil {
const priorityAndFairnessFilterHookName = "priority-and-fairness-filter"
if !s.isPostStartHookRegistered(priorityAndFairnessFilterHookName) {
err := s.AddPostStartHook(priorityAndFairnessFilterHookName, func(context PostStartHookContext) error {
genericfilters.StartPriorityAndFairnessWatermarkMaintenance(context.StopCh)
return nil
})
if err != nil {
return nil, err
}
}
} else {
const maxInFlightFilterHookName = "max-in-flight-filter"
if !s.isPostStartHookRegistered(maxInFlightFilterHookName) {
err := s.AddPostStartHook(maxInFlightFilterHookName, func(context PostStartHookContext) error {
genericfilters.StartMaxInFlightWatermarkMaintenance(context.StopCh)
return nil
})
if err != nil {
return nil, err
}
}
}
for _, delegateCheck := range delegationTarget.HealthzChecks() {
skip := false
for _, existingCheck := range c.HealthzChecks {
if existingCheck.Name() == delegateCheck.Name() {
skip = true
break
}
}
if skip {
continue
}
s.AddHealthChecks(delegateCheck)
}
s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}
// 安装API相关参数
installAPI(s, c.Config)
// use the UnprotectedHandler from the delegation target to ensure that we don't attempt to double authenticator, authorize,
// or some other part of the filter chain in delegation cases.
if delegationTarget.UnprotectedHandler() == nil && c.EnableIndex {
s.Handler.NonGoRestfulMux.NotFoundHandler(routes.IndexLister{
StatusCode: http.StatusNotFound,
PathProvider: s.listedPathProvider,
})
}
return s, nil
}
在New()方法中构建handlerChain，handlerChainBuilder是对completedConfig的BuildHandlerChainFunc函数的封装，默认实现如下：
// vendor/k8s.io/apiserver/pkg/server/config.go
// 通过装饰器模式包装apiHandler,包含认证、鉴权等一系列http filter chain,要先过这些http filter chain才访问到apiHandler
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler := filterlatency.TrackCompleted(apiHandler)
handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
handler = filterlatency.TrackStarted(handler, "authorization")
if c.FlowControl != nil {
handler = filterlatency.TrackCompleted(handler)
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, c.RequestWidthEstimator)
handler = filterlatency.TrackStarted(handler, "priorityandfairness")
} else {
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
}
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
handler = filterlatency.TrackStarted(handler, "impersonation")
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)
handler = filterlatency.TrackStarted(handler, "audit")
failedHandler := genericapifilters.Unauthorized(c.Serializer)
failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyChecker)
failedHandler = filterlatency.TrackCompleted(failedHandler)
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
handler = filterlatency.TrackStarted(handler, "authentication")
handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
// WithTimeoutForNonLongRunningRequests will call the rest of the request handling in a go-routine with the
// context with deadline. The go-routine can keep running, while the timeout logic will return a timeout to the client.
handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc)
handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyChecker,
c.LongRunningFunc, c.Serializer, c.RequestTimeout)
handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
}
handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyChecker)
handler = genericapifilters.WithWarningRecorder(handler)
handler = genericapifilters.WithCacheControl(handler)
handler = genericfilters.WithHSTS(handler, c.HSTSDirectives)
handler = genericfilters.WithHTTPLogging(handler)
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
handler = genericapifilters.WithTracing(handler, c.TracerProvider)
}
handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
handler = genericapifilters.WithRequestReceivedTimestamp(handler)
handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
handler = genericapifilters.WithAuditID(handler)
return handler
}
Default Filters流程如下：
以鉴权的WithAuthorization()为例：
// vendor/k8s.io/apiserver/pkg/endpoints/filters/authorization.go
func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler {
if a == nil {
klog.Warning("Authorization is disabled")
return handler
}
// 装饰器模式,在http.Handler外围包裹
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
ae := request.AuditEventFrom(ctx)
attributes, err := GetAuthorizerAttributes(ctx)
if err != nil {
responsewriters.InternalError(w, req, err)
return
}
// 调用Authorize方法做鉴权
authorized, reason, err := a.Authorize(ctx, attributes)
// an authorizer like RBAC could encounter evaluation errors and still allow the request, so authorizer decision is checked before error here.
if authorized == authorizer.DecisionAllow {
audit.LogAnnotation(ae, decisionAnnotationKey, decisionAllow)
audit.LogAnnotation(ae, reasonAnnotationKey, reason)
handler.ServeHTTP(w, req)
return
}
if err != nil {
audit.LogAnnotation(ae, reasonAnnotationKey, reasonError)
responsewriters.InternalError(w, req, err)
return
}
klog.V(4).InfoS("Forbidden", "URI", req.RequestURI, "Reason", reason)
audit.LogAnnotation(ae, decisionAnnotationKey, decisionForbid)
audit.LogAnnotation(ae, reasonAnnotationKey, reason)
responsewriters.Forbidden(ctx, attributes, w, req, reason, s)
})
}
HandlerFunc实现了http.Handler接口，定义如下：
// net/http/server.go
type HandlerFunc func(ResponseWriter, *Request)
func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) {
f(w, r)
}
2）、Serializer的加载和使用
HTTP报文会被转为Golang Type Instance，然后进行External Version到Internal Version的转换
API Resource的装载流程中和Serializer相关的逻辑如下：
以POST请求为例，registerResourceHandlers()方法中调用逻辑如下：
createHandler()方法代码如下：
// vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go
func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
// For performance tracking purposes.
trace := utiltrace.New("Create", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
namespace, name, err := scope.Namer.Name(req)
if err != nil {
if includeName {
// name was required, return
scope.err(err, w, req)
return
}
// otherwise attempt to look up the namespace
namespace, err = scope.Namer.Namespace(req)
if err != nil {
scope.err(err, w, req)
return
}
}
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
defer cancel()
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
if err != nil {
scope.err(err, w, req)
return
}
gv := scope.Kind.GroupVersion()
s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer)
if err != nil {
scope.err(err, w, req)
return
}
decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
if err != nil {
scope.err(err, w, req)
return
}
options := &metav1.CreateOptions{}
values := req.URL.Query()
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(values, scope.MetaGroupVersion, options); err != nil {
err = errors.NewBadRequest(err.Error())
scope.err(err, w, req)
return
}
if errs := validation.ValidateCreateOptions(options); len(errs) > 0 {
err := errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "CreateOptions"}, "", errs)
scope.err(err, w, req)
return
}
options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("CreateOptions"))
defaultGVK := scope.Kind
original := r.New()
trace.Step("About to convert to expected version")
// 1)处理请求前,使用reqScope的Serializer得到decoder进行解码
obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
if err != nil {
err = transformDecodeError(scope.Typer, err, original, gvk, body)
scope.err(err, w, req)
return
}
objGV := gvk.GroupVersion()
if !scope.AcceptsGroupVersion(objGV) {
err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%v)", objGV.String(), gv.String()))
scope.err(err, w, req)
return
}
trace.Step("Conversion done")
// On create, get name from new object if unset
if len(name) == 0 {
_, name, _ = scope.Namer.ObjectName(obj)
}
if len(namespace) == 0 && *gvk == namespaceGVK {
namespace = name
}
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
audit.LogRequestObject(ae, obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)
userInfo, _ := request.UserFrom(ctx)
trace.Step("About to store object in database")
admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
requestFunc := func() (runtime.Object, error) {
return r.Create(
ctx,
name,
obj,
rest.AdmissionToValidateObjectFunc(admit, admissionAttributes, scope),
options,
)
}
// Dedup owner references before updating managed fields
dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
if scope.FieldManager != nil {
liveObj, err := scope.Creater.New(scope.Kind)
if err != nil {
return nil, fmt.Errorf("failed to create new object (Create for %v): %v", scope.Kind, err)
}
obj = scope.FieldManager.UpdateNoErrors(liveObj, obj, managerOrUserAgent(options.FieldManager, req.UserAgent()))
admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
}
if mutatingAdmission, ok := admit.(admission.MutationInterface); ok && mutatingAdmission.Handles(admission.Create) {
if err := mutatingAdmission.Admit(ctx, admissionAttributes, scope); err != nil {
return nil, err
}
}
// Dedup owner references again after mutating admission happens
dedupOwnerReferencesAndAddWarning(obj, req.Context(), true)
result, err := requestFunc()
// If the object wasn't committed to storage because it's serialized size was too large,
// it is safe to remove managedFields (which can be large) and try again.
if isTooLargeError(err) {
if accessor, accessorErr := meta.Accessor(obj); accessorErr == nil {
accessor.SetManagedFields(nil)
result, err = requestFunc()
}
}
return result, err
})
if err != nil {
scope.err(err, w, req)
return
}
trace.Step("Object stored in database")
code := http.StatusCreated
status, ok := result.(*metav1.Status)
if ok && err == nil && status.Code == 0 {
status.Code = int32(code)
}
// 2)响应时,使用reqScope的Serializer得到encoder进行解码
transformResponseObject(ctx, scope, trace, req, w, code, outputMediaType, result)
}
}
3）、Store：对Request进行响应的业务逻辑部分
以APIGroup apps的Deployment为例：
调用Deployment对应的NewStorage()方法返回DeploymentStorage，最终调用的是Deployment对应的NewREST()方法，代码如下：
// pkg/registry/apps/deployment/storage/storage.go
func NewStorage(optsGetter generic.RESTOptionsGetter) (DeploymentStorage, error) {
deploymentRest, deploymentStatusRest, deploymentRollbackRest, err := NewREST(optsGetter)
if err != nil {
return DeploymentStorage{}, err
}
return DeploymentStorage{
Deployment: deploymentRest,
Status: deploymentStatusRest,
Scale: &ScaleREST{store: deploymentRest.Store},
Rollback: deploymentRollbackRest,
}, nil
}
type REST struct {
*genericregistry.Store
categories []string
}
func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST, *RollbackREST, error) {
store := &genericregistry.Store{
NewFunc: func() runtime.Object { return &apps.Deployment{} },
NewListFunc: func() runtime.Object { return &apps.DeploymentList{} },
DefaultQualifiedResource: apps.Resource("deployments"),
CreateStrategy: deployment.Strategy,
UpdateStrategy: deployment.Strategy,
DeleteStrategy: deployment.Strategy,
ResetFieldsStrategy: deployment.Strategy,
TableConvertor: printerstorage.TableConvertor{TableGenerator: printers.NewTableGenerator().With(printersinternal.AddHandlers)},
}
options := &generic.StoreOptions{RESTOptions: optsGetter}
if err := store.CompleteWithOptions(options); err != nil {
return nil, nil, nil, err
}
statusStore := *store
statusStore.UpdateStrategy = deployment.StatusStrategy
statusStore.ResetFieldsStrategy = deployment.StatusStrategy
return &REST{store, []string{"all"}}, &StatusREST{store: &statusStore}, &RollbackREST{store: store}, nil
}
NewREST()方法用于构造并返回REST结构体实例，包括子Object的
每一个APIObject都会有一个REST结构体，负责最终处理针对本Object的Request；而这个REST结构体大多时候通过内嵌genericregistry.Store结构体来直接复用其属性和方法，特别是内建APIObject，所以这个Store结构体包含了大多数处理逻辑。该REST结构体一般定义在APIObject相应的storage/storage.go文件中
genericregistry.Store：
genericregistry.Store的属性中定义了需要Strategy方法，它们是GenericAPIServer提供给外部的扩展点，每个APIObject都可以写自己的strategy，对创建、修改、删除等过程进行干预
genericregistry.Store包含Get、Create、Update、Delete等方法，它们会最终被RequestHandler所调用，完成请求任务。大部分内建APIObject都复用了Store，从而避免了自己写大量逻辑，也使得Strategy这种模式继续得以起效
以Create为例对应Strategy接口如下：
// vendor/k8s.io/apiserver/pkg/registry/rest/create.go
type RESTCreateStrategy interface {
runtime.ObjectTyper
// The name generator is used when the standard GenerateName field is set.
// The NameGenerator will be invoked prior to validation.
names.NameGenerator
// NamespaceScoped returns true if the object must be within a namespace.
NamespaceScoped() bool
// PrepareForCreate is invoked on create before validation to normalize
// the object. For example: remove fields that are not to be persisted,
// sort order-insensitive list fields, etc. This should not remove fields
// whose presence would be considered a validation error.
//
// Often implemented as a type check and an initailization or clearing of
// status. Clear the status because status changes are internal. External
// callers of an api (users) should not be setting an initial status on
// newly created objects.
PrepareForCreate(ctx context.Context, obj runtime.Object)
// Validate returns an ErrorList with validation errors or nil. Validate
// is invoked after default fields in the object have been filled in
// before the object is persisted. This method should not mutate the
// object.
Validate(ctx context.Context, obj runtime.Object) field.ErrorList
// WarningsOnCreate returns warnings to the client performing a create.
// WarningsOnCreate is invoked after default fields in the object have been filled in
// and after Validate has passed, before Canonicalize is called, and the object is persisted.
// This method must not mutate the object.
//
// Be brief; limit warnings to 120 characters if possible.
// Don't include a "Warning:" prefix in the message (that is added by clients on output).
// Warnings returned about a specific field should be formatted as "path.to.field: message".
// For example: `spec.imagePullSecrets[0].name: invalid empty name ""`
//
// Use warning messages to describe problems the client making the API request should correct or be aware of.
// For example:
// - use of deprecated fields/labels/annotations that will stop working in a future release
// - use of obsolete fields/labels/annotations that are non-functional
// - malformed or invalid specifications that prevent successful handling of the submitted object,
// but are not rejected by validation for compatibility reasons
//
// Warnings should not be returned for fields which cannot be resolved by the caller.
// For example, do not warn about spec fields in a subresource creation request.
WarningsOnCreate(ctx context.Context, obj runtime.Object) []string
// Canonicalize allows an object to be mutated into a canonical form. This
// ensures that code that operates on these objects can rely on the common
// form for things like comparison. Canonicalize is invoked after
// validation has succeeded but before the object has been persisted.
// This method may mutate the object. Often implemented as a type check or
// empty method.
Canonicalize(obj runtime.Object)
}
API Resource的装载流程中和Store相关的逻辑如下：
通过前面对Serializer和Admission的剖析，我们得知Http请求最终都是在诸如createHandler方法中构造出来的
例如Create（HTTP POST）就是在文件vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go中的createHandler方法中
而Update（HTTP PUT）就是在文件vendor/k8s.io/apiserver/pkg/endpoints/handlers/update.go中的UpdateResource方法中
这两处的变量r就是传入的形式参数，其实际参数就是上述Store结构体实例
14、Authentication与Authorization
1）、用户种类和用户信息
有两类用户：
- Service Account：集群内管理，主要目的是集群内程序与APIServer连通之用
- Normal User：由集群外提供，对集群来说它更像一个抽象的概念，不负责保存和维护
用户都包含如下信息，登录过程获取这些信息放入Request，在鉴权过程使用：
- Name：用户名
- UID：唯一ID
- Groups：所隶属的group
- Extra fields：一些额外的信息，因不同的登录验证策略而不同
2）、登录验证策略
一个HTTP请求到来时，先要经过登录流程。Server的Filter中一个就是做登录验证的工作
登录验证的策略有多种，如上图所示。一个Request只要被一个策略明确Approve或明确Reject，那么登录过程就结束了
系统默认有一个Anonymous策略：如果上述策略均不Approve也不Reject，那么这个Request将被赋予Anonymous身份交由Server处理
3）、鉴权模式
Kubernetes中有4种鉴权模式：
- Role Based Access Control（RBAC）：目前默认的鉴权模式，通过定义Role/ClusterRole，然后用RoleBinding/ClusterRoleBinding把它们赋予user或group
- Attribute Based Access Control（ABAC）：根据各种属性（user的、group的、目标resource的甚至是环境变量）来决定一个Request是否能做某件事情
- Node Authorization：主要对由kuberlet过来的请求进行鉴权
- Webhook Authorization：APIServer把鉴权的工作交给一个web服务，server向该服务发送SubjectAccessReview对象，该服务做出判断把结果附在SAR上传回
4）、Authentication & Authorization何时发生
HTTP请求进来先进入AggregatorServer，AggregatorServer会进行登录验证和鉴权
- 如果请求由KubeAPIServer或者APIExtensionsServer处理，通过delegation将请求分发下去，在KubeAPIServer和APIExtensionsServer不会再进行登录和鉴权校验
- 如果请求由CustomServer处理，通过Proxy将请求代理转发，CustomServer会验证Proxy转发过来的HTTP Request附带的证书信息，确认是否是AggregatorServer转发过来的，此时Request上会包含用户登录信息，CustomServer不会再进行登录验证。但是会鉴权，CustomServer会发一个SubjectAccessReview的Request给到AggregatorServer进行验证，AggregatorServer把验证后的结果附在SubjectAccessReview发回给CustomServer
一个Request进入AggregatorServer后会先经过Filter的登录和鉴权
1）触发登录和鉴权路径1：Delegate机制
文件vendor/k8s.io/apiserver/pkg/server/config.go中的New方法
问题：在Delegate这条路径上，除了在AggregatorServer上发生一次外，登录和鉴权会不会在KubeAPIServer和APIExtensionsServer上发生呢？
答案：不会，这显然是多余的验证。在形成delegation链（Server Chain）时，下一个Server的unprotectedHandler被使用，它是未经过Filter包裹的，自然不会触发登录和鉴权等（代码1)处）
2）触发登录和鉴权路径2：Proxy机制
文件vendor/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go中的AddAPIService方法
问题：在Proxy这条路径上，为什么会发生登录和鉴权呢？
答案：上图代码是在AggregatorServer内，给一个APIService对象代表的APIGroup赋予Http handler。可以看到还是交由GenericAPIServer的NonGoRestfulMux，那么每个Request到来后，Filter Chain就会被执行，从而触发其中的登录和鉴权（参见前序章节讲解Server Chain的形成部分，这个NonGoRestfulMux实际上在director背后，承接director转过来的请求，而director是经过Filter包装的）
5）、登录验证器的实现和加载
// pkg/kubeapiserver/authenticator/config.go
func (config Config) New() (authenticator.Request, *spec.SecurityDefinitions, error) {
var authenticators []authenticator.Request
var tokenAuthenticators []authenticator.Token
securityDefinitions := spec.SecurityDefinitions{}
// front-proxy, BasicAuth methods, local first, then remote
// Add the front proxy authenticator if requested
// Authenticating Proxy策略
if config.RequestHeaderConfig != nil {
requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
config.RequestHeaderConfig.CAContentProvider.VerifyOptions,
config.RequestHeaderConfig.AllowedClientNames,
config.RequestHeaderConfig.UsernameHeaders,
config.RequestHeaderConfig.GroupHeaders,
config.RequestHeaderConfig.ExtraHeaderPrefixes,
)
authenticators = append(authenticators, authenticator.WrapAudienceAgnosticRequest(config.APIAudiences, requestHeaderAuthenticator))
}
// X509 methods
// X509 Client Certs策略
if config.ClientCAContentProvider != nil {
certAuth := x509.NewDynamic(config.ClientCAContentProvider.VerifyOptions, x509.CommonNameUserConversion)
authenticators = append(authenticators, certAuth)
}
// Bearer token methods, local first, then remote
// Static Token File策略
if len(config.TokenAuthFile) > 0 {
tokenAuth, err := newAuthenticatorFromTokenFile(config.TokenAuthFile)
if err != nil {
return nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, tokenAuth))
}
// Service Account Tokens策略
if len(config.ServiceAccountKeyFiles) > 0 {
serviceAccountAuth, err := newLegacyServiceAccountAuthenticator(config.ServiceAccountKeyFiles, config.ServiceAccountLookup, config.APIAudiences, config.ServiceAccountTokenGetter)
if err != nil {
return nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
}
if len(config.ServiceAccountIssuers) > 0 {
serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountKeyFiles, config.APIAudiences, config.ServiceAccountTokenGetter)
if err != nil {
return nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
}
// Bootstrap Tokens策略
if config.BootstrapToken {
if config.BootstrapTokenAuthenticator != nil {
// TODO: This can sometimes be nil because of
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, config.BootstrapTokenAuthenticator))
}
}
// NOTE(ericchiang): Keep the OpenID Connect after Service Accounts.
//
// Because both plugins verify JWTs whichever comes first in the union experiences
// cache misses for all requests using the other. While the service account plugin
// simply returns an error, the OpenID Connect plugin may query the provider to
// update the keys, causing performance hits.
// OpenID Connect Tokens策略
if len(config.OIDCIssuerURL) > 0 && len(config.OIDCClientID) > 0 {
// TODO(enj): wire up the Notifier and ControllerRunner bits when OIDC supports CA reload
var oidcCAContent oidc.CAContentProvider
if len(config.OIDCCAFile) != 0 {
var oidcCAErr error
oidcCAContent, oidcCAErr = dynamiccertificates.NewDynamicCAContentFromFile("oidc-authenticator", config.OIDCCAFile)
if oidcCAErr != nil {
return nil, nil, oidcCAErr
}
}
oidcAuth, err := newAuthenticatorFromOIDCIssuerURL(oidc.Options{
IssuerURL: config.OIDCIssuerURL,
ClientID: config.OIDCClientID,
CAContentProvider: oidcCAContent,
UsernameClaim: config.OIDCUsernameClaim,
UsernamePrefix: config.OIDCUsernamePrefix,
GroupsClaim: config.OIDCGroupsClaim,
GroupsPrefix: config.OIDCGroupsPrefix,
SupportedSigningAlgs: config.OIDCSigningAlgs,
RequiredClaims: config.OIDCRequiredClaims,
})
if err != nil {
return nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, authenticator.WrapAudienceAgnosticToken(config.APIAudiences, oidcAuth))
}
// Webhook Token Authentication策略
if len(config.WebhookTokenAuthnConfigFile) > 0 {
webhookTokenAuth, err := newWebhookTokenAuthenticator(config)
if err != nil {
return nil, nil, err
}
tokenAuthenticators = append(tokenAuthenticators, webhookTokenAuth)
}
if len(tokenAuthenticators) > 0 {
// Union the token authenticators
tokenAuth := tokenunion.New(tokenAuthenticators...)
// Optionally cache authentication results
if config.TokenSuccessCacheTTL > 0 || config.TokenFailureCacheTTL > 0 {
tokenAuth = tokencache.New(tokenAuth, true, config.TokenSuccessCacheTTL, config.TokenFailureCacheTTL)
}
authenticators = append(authenticators, bearertoken.New(tokenAuth), websocket.NewProtocolAuthenticator(tokenAuth))
securityDefinitions["BearerToken"] = &spec.SecurityScheme{
SecuritySchemeProps: spec.SecuritySchemeProps{
Type: "apiKey",
Name: "authorization",
In: "header",
Description: "Bearer Token authentication",
},
}
}
if len(authenticators) == 0 {
if config.Anonymous {
return anonymous.NewAuthenticator(), &securityDefinitions, nil
}
return nil, &securityDefinitions, nil
}
authenticator := union.New(authenticators...)
authenticator = group.NewAuthenticatedGroupAdder(authenticator)
// Anonymous策略
if config.Anonymous {
// If the authenticator chain returns an error, return an error (don't consider a bad bearer token
// or invalid username/password combination anonymous).
authenticator = union.NewFailOnError(authenticator, anonymous.NewAuthenticator())
}
return authenticator, &securityDefinitions, nil
}
6）、鉴权器的实现和加载
// pkg/kubeapiserver/authorizer/config.go
func (config Config) New() (authorizer.Authorizer, authorizer.RuleResolver, error) {
if len(config.AuthorizationModes) == 0 {
return nil, nil, fmt.Errorf("at least one authorization mode must be passed")
}
var (
authorizers []authorizer.Authorizer
ruleResolvers []authorizer.RuleResolver
)
for _, authorizationMode := range config.AuthorizationModes {
// Keep cases in sync with constant list in k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes/modes.go.
switch authorizationMode {
// Node Authorization
case modes.ModeNode:
node.RegisterMetrics()
graph := node.NewGraph()
node.AddGraphEventHandlers(
graph,
config.VersionedInformerFactory.Core().V1().Nodes(),
config.VersionedInformerFactory.Core().V1().Pods(),
config.VersionedInformerFactory.Core().V1().PersistentVolumes(),
config.VersionedInformerFactory.Storage().V1().VolumeAttachments(),
)
nodeAuthorizer := node.NewAuthorizer(graph, nodeidentifier.NewDefaultNodeIdentifier(), bootstrappolicy.NodeRules())
authorizers = append(authorizers, nodeAuthorizer)
ruleResolvers = append(ruleResolvers, nodeAuthorizer)
case modes.ModeAlwaysAllow:
alwaysAllowAuthorizer := authorizerfactory.NewAlwaysAllowAuthorizer()
authorizers = append(authorizers, alwaysAllowAuthorizer)
ruleResolvers = append(ruleResolvers, alwaysAllowAuthorizer)
case modes.ModeAlwaysDeny:
alwaysDenyAuthorizer := authorizerfactory.NewAlwaysDenyAuthorizer()
authorizers = append(authorizers, alwaysDenyAuthorizer)
ruleResolvers = append(ruleResolvers, alwaysDenyAuthorizer)
// ABAC
case modes.ModeABAC:
abacAuthorizer, err := abac.NewFromFile(config.PolicyFile)
if err != nil {
return nil, nil, err
}
authorizers = append(authorizers, abacAuthorizer)
ruleResolvers = append(ruleResolvers, abacAuthorizer)
// Webhook Authorization
case modes.ModeWebhook:
if config.WebhookRetryBackoff == nil {
return nil, nil, errors.New("retry backoff parameters for authorization webhook has not been specified")
}
webhookAuthorizer, err := webhook.New(config.WebhookConfigFile,
config.WebhookVersion,
config.WebhookCacheAuthorizedTTL,
config.WebhookCacheUnauthorizedTTL,
*config.WebhookRetryBackoff,
config.CustomDial)
if err != nil {
return nil, nil, err
}
authorizers = append(authorizers, webhookAuthorizer)
ruleResolvers = append(ruleResolvers, webhookAuthorizer)
// RBAC
case modes.ModeRBAC:
rbacAuthorizer := rbac.New(
&rbac.RoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().Roles().Lister()},
&rbac.RoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().RoleBindings().Lister()},
&rbac.ClusterRoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoles().Lister()},
&rbac.ClusterRoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoleBindings().Lister()},
)
authorizers = append(authorizers, rbacAuthorizer)
ruleResolvers = append(ruleResolvers, rbacAuthorizer)
default:
return nil, nil, fmt.Errorf("unknown authorization mode %s specified", authorizationMode)
}
}
return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil
}
参考：
Kubernetes源码开发之旅三：API Server源码剖析