k8s Api server 源码解读

背景

     由于在工作中，需要引入k8s Resource Quota,所以这次借此契机研究Api Sever源码和resource quota的创建流程。

           K8s源码版本: v 1.10

           Golang版本: 1.10.5

           etcd： 3.0+

           ide： idea

   对于1.10及以上的k8s要求golang版本必须在1.10及以上。

        利用git下载K8s源码 本地编译启动api-server 需要执行下面命令 

         cd ./go/src/k8s.io/kubernetes

         make WHAT=cmd/kube-apiserver all 

 先启动etcd， 然后就可以在本地直接启动api-server, 进入debug模式调试。如果遇见编译nil不兼容的问题，请将nil改为0值即可。

 

类比Java的spring体系，k8s实现restful架构是通过Go-restful第三方模块实现的。

Go-restful主要名词解释：

Route: 每个请求对应一个route，通过route指定具体的逻辑处理函数。

Webservice： 多个route组成一个webservice。

Container: 多个webservice组成一个container。

直接上go-restful 官方demo, 比较直观:

package main
import (
   "github.com/emicklei/go-restful"
   "log"
   "net/http"
)


// This example shows the minimal code needed to get a restful.WebService working.

// GET http://localhost:8080/hello

//  Go restful github site: https://github.com/emicklei/go-restful

func main() {

   ws := new(restful.WebService)

   ws.Route(ws.GET("/hello").To(hello))

   ws.Route(ws.GET("/hello/world").To(world))

   restful.Add(ws)

   http.ListenAndServe(":8080", nil)

   log.Printf("start")

}

func hello(req *restful.Request, resp *restful.Response) {
   //resp.w
   resp.Write([]byte("hello world"))
}

func world(req *restful.Request, resp *restful.Response) {
   //resp.w
   resp.Write([]byte("this is go world."))

}
-----------------------------------------------------
package main

import (
"github.com/emicklei/go-restful"
"io"
"log"
"net/http"
)

func main() {
   ws := new(restful.WebService)
   ws.Route(ws.GET("/hello").To(hello))
   restful.Add(ws)
   go func() {
      log.Fatal(http.ListenAndServe(":8080", nil))
   }()

   container2 := restful.NewContainer()
   ws2 := new(restful.WebService)
   ws2.Route(ws2.GET("/hello").To(hello2))
   container2.Add(ws2)
   server := &http.Server{Addr: ":8081", Handler: container2}
   log.Fatal(server.ListenAndServe())
}

func hello(req *restful.Request, resp *restful.Response) {
   io.WriteString(resp, "default world")
}

func hello2(req *restful.Request, resp *restful.Response) {
   io.WriteString(resp, "second world")
}

 

---

所以现在的目标：

   1，找到api server如和初始化go-restful框架？

   2，找到对应的handler看k8s如和创建一个resoure quota？

   3，找到k8s如何将create的resource quota对象存储在etcd中？

---

现在我们来解决问题1. 

由k8s的 apiserver启动cmd/kube-apiserver/app中main函数：

package main

import (
   goflag "flag"
   "fmt"
   "math/rand"
   "os"
   "time"
   "github.com/spf13/pflag"
   utilflag "k8s.io/apiserver/pkg/util/flag"
   "k8s.io/apiserver/pkg/util/logs"
   "k8s.io/kubernetes/cmd/kube-apiserver/app"
   _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
   _ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
)

func main() {
   rand.Seed(time.Now().UTC().UnixNano())
   // new新的command。
   command := app.NewAPIServerCommand()
   // TODO: once we switch everything over to Cobra commands, we can go back to calling

   // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
   // normalize func and add the go flag set by hand.
   //pflog主要是命令行参数标准化 可忽略。
   pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
   pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
   // utilflag.InitFlags()
   logs.InitLogs()
   defer logs.FlushLogs()
   // 这里是入口
   if err := command.Execute(); err != nil {
      fmt.Fprintf(os.Stderr, "error: %v\n", err)
      os.Exit(1)
   }
}

command.Execute()调用的是ExcuteC()方法:

func (c *Command) ExecuteC() (cmd *Command, err error) {
   // Regardless of what command execute is called on, run on Root only
   if c.HasParent() {
      return c.Root().ExecuteC()
   }
   // windows hook
   if preExecHookFn != nil {
      preExecHookFn(c)
   }
   // initialize help as the last point possible to allow for user
   // overriding
   c.InitDefaultHelpCmd()
   var args []string
   // Workaround FAIL with "go test -v" or "cobra.test -test.v", see #155
   // 解析命令行参数
   if c.args == nil && filepath.Base(os.Args[0]) != "cobra.test" {
      args = os.Args[1:]
   } else {
      args = c.args
   }
   var flags []string
   if c.TraverseChildren {
      cmd, flags, err = c.Traverse(args)
   } else {
      cmd, flags, err = c.Find(args)
   }
   if err != nil {
      // If found parse to a subcommand and then failed, talk about the subcommand
      if cmd != nil {
         c = cmd
      }
      if !c.SilenceErrors {
         c.Println("Error:", err.Error())
         c.Printf("Run '%v --help' for usage.\n", c.CommandPath())
      }
      return c, err
   }
   cmd.commandCalledAs.called = true
   if cmd.commandCalledAs.name == "" {
      cmd.commandCalledAs.name = cmd.Name()
   }
   // 这里才是真正的入口函数调用开始new command中的run方法入口，前面只是一些参数封装。
   err = cmd.execute(flags)
   ………
  cmd.execute(flags)方法：
   if c == nil {
      return fmt.Errorf("Called Execute() on a nil Command")
   }

   if len(c.Deprecated) > 0 {
      c.Printf("Command %q is deprecated, %s\n", c.Name(), c.Deprecated)
   }
   // initialize help and version flag at the last point possible to allow for user
   // overriding
   c.InitDefaultHelpFlag()
   c.InitDefaultVersionFlag()
   err = c.ParseFlags(a)
   if err != nil {
      return c.FlagErrorFunc()(c, err)
   }

   // If help is called, regardless of other flags, return we want help.
   // Also say we need help if the command isn't runnable.
   helpVal, err := c.Flags().GetBool("help")
   if err != nil {
      // should be impossible to get here as we always declare a help
      // flag in InitDefaultHelpFlag()
      c.Println("\"help\" flag declared as non-bool. Please correct your code")
      return err
   }

   if helpVal {
      return flag.ErrHelp
   }

   // for back-compat, only add version flag behavior if version is defined
   if c.Version != "" {
      versionVal, err := c.Flags().GetBool("version")
      if err != nil {
         c.Println("\"version\" flag declared as non-bool. Please correct your code")
         return err
      }
      if versionVal {
         err := tmpl(c.OutOrStdout(), c.VersionTemplate(), c)
         if err != nil {
           c.Println(err)
         }
         return err
      }
   }

   if !c.Runnable() {
      return flag.ErrHelp
   }
   c.preRun()
   argWoFlags := c.Flags().Args()
   if c.DisableFlagParsing {
      argWoFlags = a
   }

   if err := c.ValidateArgs(argWoFlags); err != nil {
      return err
   }

   for p := c; p != nil; p = p.Parent() {
      if p.PersistentPreRunE != nil {
         if err := p.PersistentPreRunE(c, argWoFlags); err != nil {
            return err
         }
         break
      } else if p.PersistentPreRun != nil {
         p.PersistentPreRun(c, argWoFlags)
         break
      }
   }
   if c.PreRunE != nil {
      if err := c.PreRunE(c, argWoFlags); err != nil {
         return err
      }

   } else if c.PreRun != nil {
     c.PreRun(c, argWoFlags)
   }
   if err := c.validateRequiredFlags(); err != nil {
      return err
   }
   if c.RunE != nil {
      if err := c.RunE(c, argWoFlags); err != nil {
         return err
      }
   } else {
      // 在此之前，主要就是参数的解析，版本验证等预处理过程，这里才是其new command时候传入的Run方法。
     c.Run(c, argWoFlags)
   }

   …………

回到app. NewAPIServerCommand()方法:

func NewAPIServerCommand() *cobra.Command {

   // 设置默认值：注意，我们在命令行中传入的参数，在k8s中统一称之为option，经过sdk flag包处理后新建一个config对象并将option赋值给config中的属性字段
   /*
    *  MaxRequestsInFlight:         defaults.MaxRequestsInFlight, 400
    * MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,  200
    * RequestTimeout:              defaults.RequestTimeout,  60s
    * MinRequestTimeout:           defaults.MinRequestTimeout, 1800
    */

   s := options.NewServerRunOptions()
   cmd := &cobra.Command{
      Use: "kube-apiserver",
      Long: `The Kubernetes API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
      Run: func(cmd *cobra.Command, args []string) {
         verflag.PrintAndExitIfRequested()
         utilflag.PrintFlags(cmd.Flags())
         stopCh := server.SetupSignalHandler()
         // 核心入口方法
         if err := Run(s, stopCh); err != nil {
            fmt.Fprintf(os.Stderr, "%v\n", err)
            os.Exit(1)
         }
      },
   }

   // 读取命令行参数配置信息例如 admission-control,如果要开启resourcequota必须配置此参数.
   s.AddFlags(cmd.Flags())
   return cmd
}

 

进入上面标红的Run方法:

func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
   // To help debugging, immediately log version
   glog.Infof("Version: %+v", version.Get())
   server, err := CreateServerChain(runOptions, stopCh)
   if err != nil {
      return err
   }
   return server.PrepareRun().Run(stopCh)
}

进入CreateServerChain()方法:

// 这里才是apiserver自己的核心入口方法，之前的只是一系列封装和初始化。
func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) (*genericapiserver.GenericAPIServer, error) {
   // 创建代理隧道，这个没搞懂，猜测应该是master 和各个node通信会用到。 
   nodeTunneler, proxyTransport, err := CreateNodeDialer(runOptions)
   if err != nil {
      return nil, err
   }
   // 初始化api-server配置 例如集群默认的ip段配置等，初始化admission插件。
   // 第一个返回值kubeAPIServerConfig: 前面解释过命令行参数统称为option，但是真正使用的时候是config。
   // 第二个返回值sharedInformers:这里暂时不做介绍。只需要知道这个是一个缓存事件异步处理框架就可以。如kube-scheduler、kubelet等跟apiserver都是通过缓存间接通信的，SharedInformer一方面收集客户端和其它组件的请求，一方面负责通知该事件的关注者，还有它会把缓存中的数据同步到ETCD中后面在kube-controller-manager模块会详细介绍。

   // 第三个返回值versionedInformers: 同样是一个缓存器，但是以版本为区分，因为在k8s支持的api版本目前为/api/v1,因此这个暂时是硬代码，后面会看见。
   // insecureServingOptions：server的配置信息, 可忽略
   // serviceResolver：可忽略
   // pluginInitializer：这个比较重要，很多功能都是通过插件的方式提供，我们在启动api-server时候，通过--admission-       //control参数指定需要加载的插件，例如: NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,Priority.
   kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
   if err != nil {
      return nil, err
   }
   // TPRs are enabled and not yet beta, since this these are the successor, they fall under the same enablement rule
   // If additional API servers are added, they should be gated.
   // 加载自定义的一些api。可忽略
   apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, pluginInitializer, runOptions)
   if err != nil {
      return nil, err
   }
    // 加载自定义的一些api。可忽略
   apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.EmptyDelegate)
   if err != nil {
      return nil, err
   }
   // 高潮来了，通过名字也可以看出，apiserver的创建方法。
   kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers, versionedInformers)
   if err != nil {
      return nil, err
   }
   // if we're starting up a hacked up version of this API server for a weird test case,
   // just start the API server as is because clients don't get built correctly when you do this
   if len(os.Getenv("KUBE_API_VERSIONS")) > 0 {
      if insecureServingOptions != nil {
         insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(kubeAPIServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
         if err := kubeserver.NonBlockingRun(insecureServingOptions, insecureHandlerChain, kubeAPIServerConfig.GenericConfig.RequestTimeout, stopCh); err != nil {
            return nil, err
         }
      }
      return kubeAPIServer.GenericAPIServer, nil
   }
   // otherwise go down the normal path of standing the aggregator up in front of the API server
   // this wires up openapi
   kubeAPIServer.GenericAPIServer.PrepareRun()
   // This will wire up openapi for extension api server
   apiExtensionsServer.GenericAPIServer.PrepareRun()
   // aggregator comes last in the chain
   aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport, pluginInitializer)
   if err != nil {
      return nil, err
   }
   aggregatorConfig.ExtraConfig.ProxyTransport = proxyTransport
   aggregatorConfig.ExtraConfig.ServiceResolver = serviceResolver
   aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
   if err != nil {
      // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
      return nil, err
   }
   if insecureServingOptions != nil {
      insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(aggregatorServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
      if err := kubeserver.NonBlockingRun(insecureServingOptions, insecureHandlerChain, kubeAPIServerConfig.GenericConfig.RequestTimeout, stopCh); err != nil {
         return nil, err
      }
   }
   return aggregatorServer.GenericAPIServer, nil
}

 

重点分析下CreateKubeAPIServer方法:

// CreateKubeAPIServer creates and wires a workable kube-apiserver
func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory) (*master.Master, error) {
   // 这里的kubeAPIServerConfig.Complete方法可忽略，主要就是检测一些字段是否为空并，并设置默认值。
   kubeAPIServer, err := kubeAPIServerConfig.Complete(versionedInformers).New(delegateAPIServer)
   if err != nil {
      return nil, err
   }
   kubeAPIServer.GenericAPIServer.AddPostStartHook("start-kube-apiserver-informers", func(context genericapiserver.PostStartHookContext) error {
      sharedInformers.Start(context.StopCh)
      return nil
   })
   return kubeAPIServer, nil
}

kubeAPIServerConfig.Complete(versionedInformers).New 方法：

// New returns a new instance of Master from the given config.
// Certain config fields will be set to a default value if unset.
// Certain config fields must be specified, including:
//   KubeletClientConfig
func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Master, error) {
   if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) {
      return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig")
   }
   // 创建一个server struct，初始化参数apiServerHandler,swaggerConfig, container hook等
   s, err := c.GenericConfig.New("kube-apiserver", delegationTarget)
   if err != nil {
     return nil, err
   }
   // 设置log api，通过此api可以获取log文件
   if c.ExtraConfig.EnableLogsSupport {
      routes.Logs{}.Install(s.Handler.GoRestfulContainer)
   }

   m := &Master{
      GenericAPIServer: s,
   }
   // 这里就是加载核心资源对应存储设置了。
   // 目前k8s的的SchemeGroupVersion group为空，version为v1
   // 在k8s体系中，通常storage、store对应的属性都是和存储相关，这些存储包括缓存和后端的存储模块etcd，
   // k8s中的每种资源都实现了自己的存储方式。在接收到客户端请求处理时候调用预先配置的storage进行存储。
   if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) {
      legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{
         StorageFactory:             c.ExtraConfig.StorageFactory,
         ProxyTransport:             c.ExtraConfig.ProxyTransport,
         KubeletClientConfig:        c.ExtraConfig.KubeletClientConfig,
         EventTTL:                   c.ExtraConfig.EventTTL,
         ServiceIPRange:             c.ExtraConfig.ServiceIPRange,
         ServiceNodePortRange:       c.ExtraConfig.ServiceNodePortRange,
         LoopbackClientConfig:       c.GenericConfig.LoopbackClientConfig,
         ServiceAccountIssuer:       c.ExtraConfig.ServiceAccountIssuer,
         ServiceAccountAPIAudiences: c.ExtraConfig.ServiceAccountAPIAudiences,
      }
      // =======================================================================
      // =======================================================================
      // 加载api handle
      m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider)
   }

   // The order here is preserved in discovery.
   // If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"),
   // the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer.
   // This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go
   // with specific priorities.
   // TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
   // handlers that we have.
   restStorageProviders := []RESTStorageProvider{
      authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator},
      authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver},
      autoscalingrest.RESTStorageProvider{},
      batchrest.RESTStorageProvider{},
      certificatesrest.RESTStorageProvider{},
      extensionsrest.RESTStorageProvider{},
      networkingrest.RESTStorageProvider{},
      policyrest.RESTStorageProvider{},
      rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer},
      schedulingrest.RESTStorageProvider{},
      settingsrest.RESTStorageProvider{},
      storagerest.RESTStorageProvider{},
      // keep apps after extensions so legacy clients resolve the extensions versions of shared resource names.
      // See https://github.com/kubernetes/kubernetes/issues/42392
      appsrest.RESTStorageProvider{},
      admissionregistrationrest.RESTStorageProvider{},
      eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL},
   }
   m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...)
   if c.ExtraConfig.Tunneler != nil {
      m.installTunneler(c.ExtraConfig.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes())
   }

   m.GenericAPIServer.AddPostStartHookOrDie("ca-registration", c.ExtraConfig.ClientCARegistrationHook.PostStartHook)
   return m, nil
}

总结：这个new方法是核心方法之一，加载并初始化了apiserverhandler，该对象包含了restful的container，并赋值给GenericAPIServer.handler属性.

 

先分析下c.GenericConfig.New() 方法:

// New creates a new server which logically combines the handling chain with the passed server.
// name is used to differentiate for logging. The handler chain in particular can be difficult as it starts delgating.
func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*GenericAPIServer, error) {
   // The delegationTarget and the config must agree on the RequestContextMapper
   if c.Serializer == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
   }
   if c.LoopbackClientConfig == nil {
      return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
   }
   // 允许定制handler
   handlerChainBuilder := func(handler http.Handler) http.Handler {
      return c.BuildHandlerChainFunc(handler, c.Config)
   }
   // 这里很容易让人联想到，restful框架接口。
   apiServerHandler := NewAPIServerHandler(name, c.RequestContextMapper, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
   s := &GenericAPIServer{
      discoveryAddresses:     c.DiscoveryAddresses,
      LoopbackClientConfig:   c.LoopbackClientConfig,
      legacyAPIGroupPrefixes: c.LegacyAPIGroupPrefixes,
      admissionControl:       c.AdmissionControl,
      requestContextMapper:   c.RequestContextMapper,
      Serializer:             c.Serializer,
      AuditBackend:           c.AuditBackend,
      delegationTarget:       delegationTarget,
      HandlerChainWaitGroup:  c.HandlerChainWaitGroup,
      minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second,
      ShutdownTimeout:   c.RequestTimeout,
      SecureServingInfo: c.SecureServing,
      ExternalAddress:   c.ExternalAddress,
      Handler: apiServerHandler,
      listedPathProvider: apiServerHandler,
      swaggerConfig: c.SwaggerConfig,
      openAPIConfig: c.OpenAPIConfig,
      postStartHooks:         map[string]postStartHookEntry{},
      preShutdownHooks:       map[string]preShutdownHookEntry{},
      disabledPostStartHooks: c.DisabledPostStartHooks,
      healthzChecks: c.HealthzChecks,
      DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer, c.RequestContextMapper),
      enableAPIResponseCompression: c.EnableAPIResponseCompression,
   }
   for k, v := range delegationTarget.PostStartHooks() {
      s.postStartHooks[k] = v
   }
   for k, v := range delegationTarget.PreShutdownHooks() {
      s.preShutdownHooks[k] = v
   }
   genericApiServerHookName := "generic-apiserver-start-informers"
   if c.SharedInformerFactory != nil && !s.isPostStartHookRegistered(genericApiServerHookName) {
      err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
         c.SharedInformerFactory.Start(context.StopCh)
         return nil
      })
      if err != nil {
         return nil, err
      }
   }
   for _, delegateCheck := range delegationTarget.HealthzChecks() {
      skip := false
      for _, existingCheck := range c.HealthzChecks {
         if existingCheck.Name() == delegateCheck.Name() {
            skip = true
            break
         }
      }
      if skip {
         continue
      }
      s.healthzChecks = append(s.healthzChecks, delegateCheck)
   }
   s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}
   // 主要安装 swagger如果被激活,监控metrics，以及根据version装载route container。
   installAPI(s, c.Config)
   // use the UnprotectedHandler from the delegation target to ensure that we don't attempt to double authenticator, authorize,
   // or some other part of the filter chain in delegation cases.
   if delegationTarget.UnprotectedHandler() == nil && c.EnableIndex {
      s.Handler.NonGoRestfulMux.NotFoundHandler(routes.IndexLister{
         StatusCode:   http.StatusNotFound,
         PathProvider: s.listedPathProvider,
      })
   }
   return s, nil
}

 

继续分析NewAPIServerHandler()

// HandlerChainBuilderFn is used to wrap the GoRestfulContainer handler using the provided handler chain.
// It is normally used to apply filtering like authentication and authorization
// type HandlerChainBuilderFn func(apiHandler http.Handler) http.Handler

func NewAPIServerHandler(name string, contextMapper request.RequestContextMapper, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
   nonGoRestfulMux := mux.NewPathRecorderMux(name)
   if notFoundHandler != nil {
      nonGoRestfulMux.NotFoundHandler(notFoundHandler)
   }
   // 到这里可以看到了开篇所说的go-restful架构了。部分解答了问题1。继续找加载webservice和route。
   gorestfulContainer := restful.NewContainer()
   gorestfulContainer.ServeMux = http.NewServeMux()
   gorestfulContainer.Router(restful.CurlyRouter{}) // e.g. for proxy/{kind}/{name}/{*}
   gorestfulContainer.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
      logStackOnRecover(s, panicReason, httpWriter)
   })
   gorestfulContainer.ServiceErrorHandler(func(serviceErr restful.ServiceError, request *restful.Request, response *restful.Response) {
      ctx, ok := contextMapper.Get(request.Request)
      if !ok {
         responsewriters.InternalError(response.ResponseWriter, request.Request, errors.New("no context found for request"))
      }
      serviceErrorHandler(ctx, s, serviceErr, request, response)
   })

   director := director{
      name:               name,
      goRestfulContainer: gorestfulContainer,
      nonGoRestfulMux:    nonGoRestfulMux,
   }
   return &APIServerHandler{
      FullHandlerChain:   handlerChainBuilder(director),
      GoRestfulContainer: gorestfulContainer,
      NonGoRestfulMux:    nonGoRestfulMux,
      Director:           director,
   }
}

回到m.InstallLegacyAPI()方法

func (m *Master) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) {
   // // =======================================================================
   // 这里的legacyRESTStorage存储了node ip和端口的信息，重要的点是apiGroupInfo。
   // 由于代码较多这里不贴NewLegacyRESTStorage方法的代码。仅列出apiGroupInfo的strcut
   // apiGroupInfo := genericapiserver.APIGroupInfo{
   //  GroupMeta:                    *legacyscheme.Registry.GroupOrDie(api.GroupName),
   //  VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
   //  Scheme:               legacyscheme.Scheme,
   //  ParameterCodec:       legacyscheme.ParameterCodec,
   //  NegotiatedSerializer: legacyscheme.Codecs,
   //  }
   //  其中VersionedResourcesStorageMap是个map， key为version的版本号，目前k8s的资源对应的版本号为v1，所以这个版本号为硬编码。
   //  value同样是一个map，该map的key为资源名，value为该资源对应的storage。每种资源都必须有一个storage用以解释资源如何存储。
   //  例如 mapStorage[“resource quota”]=resourcequotaStorage
   //   VersionedResourcesStorageMap[“v1”] = mapStorage
   legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter)
   if err != nil {
      glog.Fatalf("Error building core storage: %v", err)
   }
   if c.ExtraConfig.EnableCoreControllers {
      controllerName := "bootstrap-controller"
      coreClient := coreclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
      bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient, coreClient)
      m.GenericAPIServer.AddPostStartHookOrDie(controllerName, bootstrapController.PostStartHook)
      m.GenericAPIServer.AddPreShutdownHookOrDie(controllerName, bootstrapController.PreShutdownHook)
   }
   if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil {
      glog.Fatalf("Error in registering group versions: %v", err)
   }
}

 

这里看下继续深入下去m.GenericAPIServer.InstallLegacyAPIGroup()

func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
   if !s.legacyAPIGroupPrefixes.Has(apiPrefix) {
      return fmt.Errorf("%q is not in the allowed legacy API prefixes: %v", apiPrefix, s.legacyAPIGroupPrefixes.List())
   }
   if err := s.installAPIResources(apiPrefix, apiGroupInfo); err != nil {
      return err
   }
   // setup discovery
   apiVersions := []string{}
   for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {
      apiVersions = append(apiVersions, groupVersion.Version)
   }
   // Install the version handler.
   // Add a handler at / to enumerate the supported api versions.
   // 将所有的rest api 放到container内部
   s.Handler.GoRestfulContainer.Add(discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix, apiVersions, s.requestContextMapper).WebService())
   return nil
}

 

继续看下s.installAPIResources():

func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
   for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {
      if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
         glog.Warningf("Skipping API %v because it has no resources.", groupVersion)
         continue
      }
      // 这里重新封装了apiGroupVersion，遍历了apiGroupInfo.VersionedResourcesStorageMap生成新的map[string]storage
      apiGroupVersion := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix)
      if apiGroupInfo.OptionsExternalVersion != nil {
         apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
      }
      if err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer); err != nil {
         return fmt.Errorf("Unable to setup API %v: %v", apiGroupInfo, err)
      }
   }
   return nil
}

 

继续看apiGroupVersion.InstallREST()

// InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
// It is expected that the provided path root prefix will serve all operations. Root MUST NOT end
// in a slash.
func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
   prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version)
   installer := &APIInstaller{
      group:                        g,
      prefix:                       prefix,
      minRequestTimeout:            g.MinRequestTimeout,
      enableAPIResponseCompression: g.EnableAPIResponseCompression,
   }
   // 注意第二个参数ws便是restful webservice
   apiResources, ws, registrationErrors := installer.Install()
   versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, staticLister{apiResources}, g.Context)
   versionDiscoveryHandler.AddToWebService(ws)
   // 将生成的ws放进container中.
   container.Add(ws)
   return utilerrors.NewAggregate(registrationErrors)
}

继续installer.Install()

    

// Install handlers for API resources.
func (a *APIInstaller) Install() ([]metav1.APIResource, *restful.WebService, []error) {
   var apiResources []metav1.APIResource
   var errors []error
   //  这里我们就非常的熟悉了，new restful webService
   ws := a.newWebService()

   // Register the paths in a deterministic (sorted) order to get a deterministic swagger spec.
   // 获取map[string]storage所有的key值
   paths := make([]string, len(a.group.Storage))
   var i int = 0
   for path := range a.group.Storage {
      paths[i] = path
      i++
   }
   sort.Strings(paths)
   for _, path := range paths {
      if path == "resourcequotas" {
         fmt.Print("get resourcequotas")
      }
      // 这里a.group 实际是APIGroupVersion对象
      // 针对每个resource 生成一个新的apiResource
      // 这个方法里肯定会有对应的go-restful的Route。
      apiResource, err := a.registerResourceHandlers(path, a.group.Storage[path], ws)
      if err != nil {
         errors = append(errors, fmt.Errorf("error in registering resource: %s, %v", path, err))
      }
      if apiResource != nil {
         apiResources = append(apiResources, *apiResource)
      }
   }
   return apiResources, ws, errors
}

继续a.registerResourceHandlers(): 这个方法实在太长了，只列出关键点

func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, error) {
   。。。。。。
   。。。。
   。。
   // what verbs are supported by the storage, used to know what verbs we support per path
   // 验证是否实现Create接口
   // 这里的storage实际就是之前我们在map[string]storage中的存储接口。
   creater, isCreater := storage.(rest.Creater)
   // 验证是否实现NamedCreater接口
   namedCreater, isNamedCreater := storage.(rest.NamedCreater)
   lister, isLister := storage.(rest.Lister)
   getter, isGetter := storage.(rest.Getter)
   getterWithOptions, isGetterWithOptions := storage.(rest.GetterWithOptions)
   gracefulDeleter, isGracefulDeleter := storage.(rest.GracefulDeleter)
   collectionDeleter, isCollectionDeleter := storage.(rest.CollectionDeleter)
   updater, isUpdater := storage.(rest.Updater)
   patcher, isPatcher := storage.(rest.Patcher)
   watcher, isWatcher := storage.(rest.Watcher)
   connecter, isConnecter := storage.(rest.Connecter)
   storageMeta, isMetadata := storage.(rest.StorageMetadata)
   。。。。
   。。。。
   // Get the list of actions for the given scope.
   //k8s 资源分为两类，一类有没有namespace root类型资源， 另一类是有namespaces资，
   // 针对不对类型的资源处理方式不同。
   switch scope.Name() {
   case meta.RESTScopeNameRoot:
     。。。。。。。。。。。。
     。。。。。。。。。。。。
     。。。。。。。。。。。。

   // 这里我们以resourcequota为例子，是有namespaces之分的。所以代码分支会走到这里
   case meta.RESTScopeNameNamespace:
      // Handler for standard REST verbs (GET, PUT, POST and DELETE).
      namespaceParam := ws.PathParameter(scope.ArgumentName(), scope.ParamDescription()).DataType("string")
     namespacedPath := scope.ParamName() + "/{" + scope.ArgumentName() + "}/" + resource
      namespaceParams := []*restful.Parameter{namespaceParam}
      resourcePath := namespacedPath
      resourceParams := namespaceParams
      itemPath := namespacedPath + "/{name}"
      nameParams := append(namespaceParams, nameParam)
      proxyParams := append(nameParams, pathParam)
      itemPathSuffix := ""
      if hasSubresource {
         itemPathSuffix = "/" + subresource
         itemPath = itemPath + itemPathSuffix
         resourcePath = itemPath
         resourceParams = nameParams
      }
      apiResource.Name = path
      apiResource.Namespaced = true
      apiResource.Kind = resourceKind
      namer := handlers.ContextBasedNaming{
         GetContext:         ctxFn,
         SelfLinker:         a.group.Linker,
         ClusterScoped:      false,
         SelfLinkPathPrefix: gpath.Join(a.prefix, scope.ParamName()) + "/",
         SelfLinkPathSuffix: itemPathSuffix,
      }
      actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)
      actions = appendIf(actions, action{"POST", resourcePath, resourceParams, namer, false}, isCreater)
      actions = appendIf(actions, action{"DELETECOLLECTION", resourcePath, resourceParams, namer, false}, isCollectionDeleter)
      // DEPRECATED
      actions = appendIf(actions, action{"WATCHLIST", "watch/" + resourcePath, resourceParams, namer, false}, allowWatchList)
      actions = appendIf(actions, action{"GET", itemPath, nameParams, namer, false}, isGetter)
      if getSubpath {
         actions = appendIf(actions, action{"GET", itemPath + "/{path:*}", proxyParams, namer, false}, isGetter)
      }
      actions = appendIf(actions, action{"PUT", itemPath, nameParams, namer, false}, isUpdater)
      actions = appendIf(actions, action{"PATCH", itemPath, nameParams, namer, false}, isPatcher)
      actions = appendIf(actions, action{"DELETE", itemPath, nameParams, namer, false}, isGracefulDeleter)
      actions = appendIf(actions, action{"WATCH", "watch/" + itemPath, nameParams, namer, false}, isWatcher)
      actions = appendIf(actions, action{"CONNECT", itemPath, nameParams, namer, false}, isConnecter)
      actions = appendIf(actions, action{"CONNECT", itemPath + "/{path:*}", proxyParams, namer, false}, isConnecter && connectSubpath)
      // list or post across namespace.
      // For ex: LIST all pods in all namespaces by sending a LIST request at /api/apiVersion/pods.
      // TODO: more strongly type whether a resource allows these actions on "all namespaces" (bulk delete)
      if !hasSubresource {
         actions = appendIf(actions, action{"LIST", resource, params, namer, true}, isLister)
         actions = appendIf(actions, action{"WATCHLIST", "watch/" + resource, params, namer, true}, allowWatchList)
      }
      break
   default:
      return nil, fmt.Errorf("unsupported restscope: %s", scope.Name())
   }

   // Create Routes for the actions.
   // TODO: Add status documentation using Returns()
   // Errors (see api/errors/errors.go as well as go-restful router):
   // http.StatusNotFound, http.StatusMethodNotAllowed,
   // http.StatusUnsupportedMediaType, http.StatusNotAcceptable,
   // http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden,
   // http.StatusRequestTimeout, http.StatusConflict, http.StatusPreconditionFailed,
   // http.StatusUnprocessableEntity, http.StatusInternalServerError,
   // http.StatusServiceUnavailable
   // and api error codes
   // Note that if we specify a versioned Status object here, we may need to
   // create one for the tests, also
   // Success:
   // http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent
   //
   // test/integration/auth_test.go is currently the most comprehensive status code test
   mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer)
   allMediaTypes := append(mediaTypes, streamMediaTypes...)
   ws.Produces(allMediaTypes...)
   kubeVerbs := map[string]struct{}{}
   reqScope := handlers.RequestScope{
      ContextFunc:     ctxFn,
      Serializer:      a.group.Serializer,
      ParameterCodec:  a.group.ParameterCodec,
      Creater:         a.group.Creater,
      Convertor:       a.group.Convertor,
      Defaulter:       a.group.Defaulter,
      Typer:           a.group.Typer,
      UnsafeConvertor: a.group.UnsafeConvertor,
      // TODO: Check for the interface on storage
      TableConvertor: tableProvider,
      // TODO: This seems wrong for cross-group subresources. It makes an assumption that a subresource and its parent are in the same group version. Revisit this.
      Resource:    a.group.GroupVersion.WithResource(resource),
      Subresource: subresource,
      Kind:        fqKindToRegister,
      MetaGroupVersion: metav1.SchemeGroupVersion,
   }

  if a.group.MetaGroupVersion != nil {
      reqScope.MetaGroupVersion = *a.group.MetaGroupVersion
   }
   // k8s将http的不同类型的method，如get，post称之为verbs
   // 在这里处理经过actions。我们按照开篇的问题1，直接找post处理方式。
   for _, action := range actions {
      。。。。。
      。。。。。
      。。。。。
      // 毫无疑问，当我们创建resource quota、pod等有namespace的资源都由此分支注册的route进行处理。
      case "POST": // Create a resource.
         var handler restful.RouteFunction
         if isNamedCreater {
           //这个handler才是我们处理request请求最终函数。需要十二分注意，后面会继续分析此handler
            handler = restfulCreateNamedResource(namedCreater, reqScope, a.group.Typer, admit)
         } else {
            //这个handler才是我们处理request请求最终函数
            handler = restfulCreateResource(creater, reqScope, a.group.Typer, admit)
         }
         /**
          *  post request处理类，接收post请求并处理。InstrumentRouteFunc方法是一个简单的包装类，
          *  目的是添加一些在request基础上添加一些monitor对象。可忽略。
          */
         handler = metrics.InstrumentRouteFunc(action.Verb, resource, subresource, requestScope, handler)
         article := getArticleForNoun(kind, " ")
         doc := "create" + article + kind
         if hasSubresource {
            doc = "create " + subresource + " of" + article + kind
         }
         // 终于见到了开篇的restful route，处理post
         route := ws.POST(action.Path).To(handler).
           Doc(doc).
            Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
            Operation("create"+namespaced+kind+strings.Title(subresource)+operationSuffix).
            Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
            Returns(http.StatusOK, "OK", producedObject).
            // TODO: in some cases, the API may return a v1.Status instead of the versioned object
            // but currently go-restful can't handle multiple different objects being returned.
            Returns(http.StatusCreated, "Created", producedObject).
            Returns(http.StatusAccepted, "Accepted", producedObject).
            Reads(defaultVersionedObject).
            Writes(producedObject)
         addParams(route, action.Params)
         // 多个route先放进数组中，后面统一处理。
         routes = append(routes, route)
      case "DELETE": // Delete a resource.
        。。。。。
        。。。。。
        。。。。。



      // 将Routes数组中route逐个放到restful webservice中。
      for _, route := range routes {
         route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{
            Group:   reqScope.Kind.Group,
            Version: reqScope.Kind.Version,
            Kind:    reqScope.Kind.Kind,
         })
         route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb))
         ws.Route(route)
      }
      // Note: update GetAuthorizerAttributes() when adding a custom handler.
   }

   apiResource.Verbs = make([]string, 0, len(kubeVerbs))
   for kubeVerb := range kubeVerbs {
      apiResource.Verbs = append(apiResource.Verbs, kubeVerb)
   }
   sort.Strings(apiResource.Verbs)
   if shortNamesProvider, ok := storage.(rest.ShortNamesProvider); ok {
      apiResource.ShortNames = shortNamesProvider.ShortNames()
   }
   if categoriesProvider, ok := storage.(rest.CategoriesProvider); ok {
      apiResource.Categories = categoriesProvider.Categories()
   }
   if gvkProvider, ok := storage.(rest.GroupVersionKindProvider); ok {
      gvk := gvkProvider.GroupVersionKind(a.group.GroupVersion)
      apiResource.Group = gvk.Group
      apiResource.Version = gvk.Version
      apiResource.Kind = gvk.Kind
   }
   return &apiResource, nil
}

 

总结：api server如和初始化go-restful框架？

    

接上图

 

---

问题2:

   由上面我们可以知道处理post请求的handler是由restfulCreateNamedResource/restfulCreateResource方法创建。现在来研究下这个函数的是怎么创建handler 的

这里我们还是以resourquota为例子，由于创建resourcequota的api不能包含name变量，因此这里调用restfulCreateResource函数。

restfulCreateResource 函数：

// 注意这里的第一个参数 r rest.Creater

func restfulCreateResource(r rest.Creater, scope handlers.RequestScope, typer runtime.ObjectTyper, admit admission.Interface) restful.RouteFunction {
   return func(req *restful.Request, res *restful.Response) {
      handlers.CreateResource(r, scope, typer, admit)(res.ResponseWriter, req.Request)
   }
}
继续分析 handlers.CreateResource(r, scope, typer, admit)(res.ResponseWriter, req.Request) 
// CreateResource returns a function that will handle a resource creation.
func CreateResource(r rest.Creater, scope RequestScope, typer runtime.ObjectTyper, admission admission.Interface) http.HandlerFunc {
    // 还记得之前的NameCreater吗？ 这里实际做了一个转换。最终都是调用NameCreater方法
   return createHandler(&namedCreaterAdapter{r}, scope, typer, admission, false)
}

 

继续分析createHandler(&namedCreaterAdapter{r}, scope, typer, admission, false)

func createHandler(r rest.NamedCreater, scope RequestScope, typer runtime.ObjectTyper, admit admission.Interface, includeName bool) http.HandlerFunc {
   return func(w http.ResponseWriter, req *http.Request) {
      // For performance tracking purposes.
      trace := utiltrace.New("Create " + req.URL.Path)
      defer trace.LogIfLong(500 * time.Millisecond)
      // TODO: we either want to remove timeout or document it (if we document, move timeout out of this function and declare it in api_installer)
      timeout := parseTimeout(req.URL.Query().Get("timeout"))
      var (
         namespace, name string
         err             error
      )
      if includeName {
         namespace, name, err = scope.Namer.Name(req)
      } else {
         namespace, err = scope.Namer.Namespace(req)
      }
      if err != nil {
         scope.err(err, w, req)
         return
      }
      ctx := scope.ContextFunc(req)
      ctx = request.WithNamespace(ctx, namespace)
      gv := scope.Kind.GroupVersion()
      s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer)
      if err != nil {
         scope.err(err, w, req)
         return
      }
      // decoder主要是反序列化,将数据装载到指定的对象中。
      decoder := scope.Serializer.DecoderToVersion(s.Serializer, schema.GroupVersion{Group: gv.Group, Version: runtime.APIVersionInternal})
      body, err := readBody(req)
      if err != nil {
         scope.err(err, w, req)
         return
      }
      defaultGVK := scope.Kind
      // r为NameCreater
      original := r.New() // 创建一个实例对象
      trace.Step("About to convert to expected version")
      obj, gvk, err := decoder.Decode(body, &defaultGVK, original) // 序列化
      if err != nil {
         err = transformDecodeError(typer, err, original, gvk, body)
         scope.err(err, w, req)
         return
      }
      if gvk.GroupVersion() != gv {
         err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%v)", gvk.GroupVersion().String(), gv.String()))
         scope.err(err, w, req)
         return
      }
      trace.Step("Conversion done")
      ae := request.AuditEventFrom(ctx)
      audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
      userInfo, _ := request.UserFrom(ctx)
      admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, userInfo)
      if mutatingAdmission, ok := admit.(admission.MutationInterface); ok && mutatingAdmission.Handles(admission.Create) {
         err = mutatingAdmission.Admit(admissionAttributes)
         if err != nil {
            scope.err(err, w, req)
            return
         }
      }
      // TODO: replace with content type negotiation?
      includeUninitialized := req.URL.Query().Get("includeUninitialized") == "1"
      trace.Step("About to store object in database")
      result, err := finishRequest(timeout, func() (runtime.Object, error) {
         // ****************************************************
         // *****************重要 重要**********************
         // ****************************************************
         // ****************************************************
         // ****************************************************
         // 最终落盘到etcd中实际还是调用了rest.creater中的Create方法。
         return r.Create(
            ctx,
            name,
            obj,
            rest.AdmissionToValidateObjectFunc(admit, admissionAttributes),
            includeUninitialized,
         )
      })
      if err != nil {
         scope.err(err, w, req)
         return
      }
      trace.Step("Object stored in database")
      requestInfo, ok := request.RequestInfoFrom(ctx)
      if !ok {
         scope.err(fmt.Errorf("missing requestInfo"), w, req)
         return
      }
      if err := setSelfLink(result, requestInfo, scope.Namer); err != nil {
         scope.err(err, w, req)
         return
      }
      trace.Step("Self-link added")
      // If the object is partially initialized, always indicate it via StatusAccepted
      code := http.StatusCreated
      if accessor, err := meta.Accessor(result); err == nil {
         if accessor.GetInitializers() != nil {
            code = http.StatusAccepted
         }
      }
      status, ok := result.(*metav1.Status)
      if ok && err == nil && status.Code == 0 {
         status.Code = int32(code)
      }
      transformResponseObject(ctx, scope, req, w, code, result)
   }
}

 

经过上面分析可以得出结论实际上new对象是通过，r.New() ，create对象是通过r.Create

不论是NameCreate，还是Create实际都是都是在registerResourceHandlers()方法中创建。参考之前的分析。

 creater, isCreater := storage.(rest.Creater)。

可以看出实际create 就是storage对象。而storage对象

在NewLegacyRESTStorage时候就已经传入了。

 

所以重点分析下resourceQuotaStorer如何创建的。回到NewLegacyRESTStorage方法找到生成resourcequota的方法

resourceQuotaStorage, resourceQuotaStatusStorage := resourcequotastore.NewREST(restOptionsGetter)

// 可以看出这里的主要就是genericregistry.Store这个结构体
// 以及store.CompleteWithOptions方法。
func NewREST(optsGetter generic.RESTOptionsGetter) (*REST, *StatusREST) {
   store := &genericregistry.Store{
      NewFunc:                  func() runtime.Object { return &api.ResourceQuota{} },
      NewListFunc:              func() runtime.Object { return &api.ResourceQuotaList{} },
      DefaultQualifiedResource: api.Resource("resourcequotas"),
      CreateStrategy:      resourcequota.Strategy,
      UpdateStrategy:      resourcequota.Strategy,
      DeleteStrategy:      resourcequota.Strategy,
      ReturnDeletedObject: true,
   }
   options := &generic.StoreOptions{RESTOptions: optsGetter}
   if err := store.CompleteWithOptions(options); err != nil {
      panic(err) // TODO: Propagate error up
   }
   statusStore := *store
   statusStore.UpdateStrategy = resourcequota.StatusStrategy
   return &REST{store}, &StatusREST{store: &statusStore}
}

这里REST实际是store的一个子类(go语言继承)

我们重点关注因此r.New()实际调用了store.New()方法

store.New()方法

// 很简单实际就是直接调用了NewFun。而这个就是上面初始化store时候传入的，返回一个空值resourcequota结构体。可以类比下其他资源都是一个道理。空值主要是handler序列化时候使用。

func (e *Store) New() runtime.Object {
   return e.NewFunc()
}

再来看下r.Create方法，实际就是调用了store.Create方法。

// Create inserts a new item according to the unique key from the object.

func (e *Store) Create(ctx genericapirequest.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, includeUninitialized bool) (runtime.Object, error) {
   if err := rest.BeforeCreate(e.CreateStrategy, ctx, obj); err != nil {
      return nil, err
   }

   // at this point we have a fully formed object.  It is time to call the validators that the apiserver
   // handling chain wants to enforce.
   if createValidation != nil {
      if err := createValidation(obj.DeepCopyObject()); err != nil {
         return nil, err
      }
   }

   name, err := e.ObjectNameFunc(obj)
   if err != nil {
      return nil, err
   }
   // 这里的KeyFunc可以看下各个资源是以什么可以存储到etcd中。等下分析
   key, err := e.KeyFunc(ctx, name)
   if err != nil {
      return nil, err
   }
   qualifiedResource := e.qualifiedResourceFromContext(ctx)
   ttl, err := e.calculateTTL(obj, 0, false)
   if err != nil {
      return nil, err
   }
   // 这里才是真正存储到etcd中。但是我们还是没有找到e.Storag是什么东西 ？
   out := e.NewFunc()
   if err := = e.Storage.Create(ctx, key, obj, out, ttl); err != nil {
      err = storeerr.InterpretCreateError(err, qualifiedResource, name)
      err = rest.CheckGeneratedNameError(e.CreateStrategy, err, obj)
      if !kubeerr.IsAlreadyExists(err) {
         return nil, err
      }
      if errGet := e.Storage.Get(ctx, key, "", out, false); errGet != nil {
         return nil, err
      }
      accessor, errGetAcc := meta.Accessor(out)
      if errGetAcc != nil {
         return nil, err
      }
      if accessor.GetDeletionTimestamp() != nil {
         msg := &err.(*kubeerr.StatusError).ErrStatus.Message
         *msg = fmt.Sprintf("object is being deleted: %s", *msg)
      }
      return nil, err
   }
   if e.AfterCreate != nil {
      if err := e.AfterCreate(out); err != nil {
         return nil, err
      }
   }

   if e.Decorator != nil {
      if err := e.Decorator(obj); err != nil {
         return nil, err
      }
   }
   if !includeUninitialized {
      return e.WaitForInitialized(ctx, out)
   }
   return out, nil
}

对于上面的分析，我们找到了e.storage。但是什么时候初始化的还是不知道。还记得方法CompleteWithOptions()

func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
   if e.DefaultQualifiedResource.Empty() {
      return fmt.Errorf("store %#v must have a non-empty qualified resource", e)
   }
   if e.NewFunc == nil {
      return fmt.Errorf("store for %s must have NewFunc set", e.DefaultQualifiedResource.String())
   }
   if e.NewListFunc == nil {
      return fmt.Errorf("store for %s must have NewListFunc set", e.DefaultQualifiedResource.String())
   }
   if (e.KeyRootFunc == nil) != (e.KeyFunc == nil) {
      return fmt.Errorf("store for %s must set both KeyRootFunc and KeyFunc or neither", e.DefaultQualifiedResource.String())
   }
   var isNamespaced bool
   switch {
   case e.CreateStrategy != nil:
      isNamespaced = e.CreateStrategy.NamespaceScoped()
   case e.UpdateStrategy != nil:
      isNamespaced = e.UpdateStrategy.NamespaceScoped()
   default:
      return fmt.Errorf("store for %s must have CreateStrategy or UpdateStrategy set", e.DefaultQualifiedResource.String())

   }
   if e.DeleteStrategy == nil {
      return fmt.Errorf("store for %s must have DeleteStrategy set", e.DefaultQualifiedResource.String())
   }
   if options.RESTOptions == nil {
      return fmt.Errorf("options for %s must have RESTOptions set", e.DefaultQualifiedResource.String())
   }
   attrFunc := options.AttrFunc
   if attrFunc == nil {
      if isNamespaced {
         attrFunc = storage.DefaultNamespaceScopedAttr
      } else {
         attrFunc = storage.DefaultClusterScopedAttr
      }
   }
   if e.PredicateFunc == nil {
      e.PredicateFunc = func(label labels.Selector, field fields.Selector) storage.SelectionPredicate {
        return storage.SelectionPredicate{
            Label:    label,
            Field:    field,
            GetAttrs: attrFunc,
         }
      }
   }
   opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource)
   if err != nil {
      return err
   }

   // ResourcePrefix must come from the underlying factory
   prefix := opts.ResourcePrefix
   if !strings.HasPrefix(prefix, "/") {
      prefix = "/" + prefix
   }
   if prefix == "/" {
      return fmt.Errorf("store for %s has an invalid prefix %q", e.DefaultQualifiedResource.String(), opts.ResourcePrefix)
   }
   // Set the default behavior for storage key generation
   if e.KeyRootFunc == nil && e.KeyFunc == nil {
      if isNamespaced {
         e.KeyRootFunc = func(ctx genericapirequest.Context) string {
           return NamespaceKeyRootFunc(ctx, prefix)
         }
         // 对于包含namespace的资源实际采用的是NamespaceKeyFunc来产生存储key值的。回答了之前提出的问题。不深入讨论
         e.KeyFunc = func(ctx genericapirequest.Context, name string) (string, error) {
            return NamespaceKeyFunc(ctx, prefix, name)
         }
      } else {
         e.KeyRootFunc = func(ctx genericapirequest.Context) string {
            return prefix
         }
         e.KeyFunc = func(ctx genericapirequest.Context, name string) (string, error) {
            return NoNamespaceKeyFunc(ctx, prefix, name)
         }  }  }
   // We adapt the store's keyFunc so that we can use it with the StorageDecorator
   // without making any assumptions about where objects are stored in etcd
   keyFunc := func(obj runtime.Object) (string, error) {
      accessor, err := meta.Accessor(obj)
      if err != nil {
         return "", err
      }
      if isNamespaced {
         return e.KeyFunc(genericapirequest.WithNamespace(genericapirequest.NewContext(), accessor.GetNamespace()), accessor.GetName())
      }
      return e.KeyFunc(genericapirequest.NewContext(), accessor.GetName())
   }
   triggerFunc := options.TriggerFunc
   if triggerFunc == nil {
      triggerFunc = storage.NoTriggerPublisher
   }
   if e.DeleteCollectionWorkers == 0 {
      e.DeleteCollectionWorkers = opts.DeleteCollectionWorkers
   }
   e.EnableGarbageCollection = opts.EnableGarbageCollection
   if e.ObjectNameFunc == nil {
      e.ObjectNameFunc = func(obj runtime.Object) (string, error) {
         accessor, err := meta.Accessor(obj)
        if err != nil {
            return "", err
         }
         return accessor.GetName(), nil
      }
   }
   // 这里初始化了e.Storage
   if e.Storage == nil {
      e.Storage, e.DestroyFunc = opts.Decorator(
         opts.StorageConfig,
         e.NewFunc(),
         prefix,
         keyFunc,
         e.NewListFunc,
         attrFunc,
         triggerFunc,
      )
      if opts.CountMetricPollPeriod > 0 {
         stopFunc := e.startObservingCount(opts.CountMetricPollPeriod)
         previousDestroy := e.DestroyFunc
         e.DestroyFunc = func() {
            stopFunc()
            if previousDestroy != nil {
               previousDestroy() 
             }           
         }
      }
   }
   return nil
}

 

这里看下 opts.Decorator到底做了什么之前，必须先来看看opts是什么东西。这里分析下GetRESTOptions源码：

// SimpleRestOptionsFactory是在一开始就通过option 传入的。这里不做讨论。不是重点。

func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
   ret := generic.RESTOptions{
      StorageConfig:           &f.Options.StorageConfig,
      Decorator:               generic.UndecoratedStorage,
      EnableGarbageCollection: f.Options.EnableGarbageCollection,
      DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers,
      ResourcePrefix:          resource.Group + "/" + resource.Resource,
      CountMetricPollPeriod:   f.Options.StorageConfig.CountMetricPollPeriod,
   }
   if f.Options.EnableWatchCache {
      sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
      if err != nil {
         return generic.RESTOptions{}, err
      }
      cacheSize, ok := sizes[resource]
      if !ok {
         cacheSize = f.Options.DefaultWatchCacheSize
      }
      // 由于默认都是激活了，缓存，所以这里的Decotator实际是genericregistry.StorageWithCache函数返回值控制。
      ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
   }
   return ret, nil

}

StorageWithCache函数:
 

func StorageWithCacher(capacity int) generic.StorageDecorator {
   return func(
      storageConfig *storagebackend.Config,
      objectType runtime.Object,
      resourcePrefix string,
      keyFunc func(obj runtime.Object) (string, error),
      newListFunc func() runtime.Object,
      getAttrsFunc storage.AttrFunc,
      triggerFunc storage.TriggerPublisherFunc) (storage.Interface, factory.DestroyFunc) {
      // 这里实际初始化了etcd后端存储。
      s, d := generic.NewRawStorage(storageConfig)
      if capacity == 0 {
         glog.V(5).Infof("Storage caching is disabled for %T", objectType)
         return s, d
      }
      glog.V(5).Infof("Storage caching is enabled for %T with capacity %v", objectType, capacity)
      // TODO: we would change this later to make storage always have cacher and hide low level KV layer inside.
      // Currently it has two layers of same storage interface -- cacher and low level kv.
      cacherConfig := storage.CacherConfig{
         CacheCapacity:        capacity,
         Storage:              s,
         Versioner:            etcdstorage.APIObjectVersioner{},
         Type:                 objectType,
         ResourcePrefix:       resourcePrefix,
         KeyFunc:              keyFunc,
         NewListFunc:          newListFunc,
         GetAttrsFunc:         getAttrsFunc,
         TriggerPublisherFunc: triggerFunc,
         Codec:                storageConfig.Codec,
      }
      // 看以看出实际是个cache包装类。
      cacher := storage.NewCacherFromConfig(cacherConfig)
      destroyFunc := func() {
         cacher.Stop()
         d()
      }
      // TODO : Remove RegisterStorageCleanup below when PR
      // https://github.com/kubernetes/kubernetes/pull/50690
      // merges as that shuts down storage properly
      RegisterStorageCleanup(destroyFunc)
      return cacher, destroyFunc
   }
}

在看下storage.NewCacherFromConfig方法

func NewCacherFromConfig(config CacherConfig) *Cacher {
   watchCache := newWatchCache(config.CacheCapacity, config.KeyFunc, config.GetAttrsFunc)
   listerWatcher := newCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc)
   reflectorName := "storage/cacher.go:" + config.ResourcePrefix
   // Give this error when it is constructed rather than when you get the
   // first watch item, because it's much easier to track down that way.
   if obj, ok := config.Type.(runtime.Object); ok {
      if err := runtime.CheckCodec(config.Codec, obj); err != nil {
         panic("storage codec doesn't seem to match given type: " + err.Error())
      }
   }
   stopCh := make(chan struct{})
   // 划重点
   cacher := &Cacher{
      ready:       newReady(),    
      storage:     config.Storage,
      objectType:  reflect.TypeOf(config.Type),
      watchCache:  watchCache,
      reflector:   cache.NewNamedReflector(reflectorName, listerWatcher, config.Type, watchCache, 0),
      versioner:   config.Versioner,
      triggerFunc: config.TriggerPublisherFunc,
      watcherIdx:  0,
      watchers: indexedWatchers{
         allWatchers:   make(map[int]*cacheWatcher),
         valueWatchers: make(map[string]watchersMap),
      },
      // TODO: Figure out the correct value for the buffer size.
      incoming:              make(chan watchCacheEvent, 100),
      dispatchTimeoutBudget: newTimeBudget(stopCh),
      // We need to (potentially) stop both:
      // - wait.Until go-routine
      // - reflector.ListAndWatch
      // and there are no guarantees on the order that they will stop.
      // So we will be simply closing the channel, and synchronizing on the WaitGroup.
      stopCh: stopCh,
   }
   watchCache.SetOnEvent(cacher.processEvent)
   go cacher.dispatchEvents()
   cacher.stopWg.Add(1)
   go func() {
      defer cacher.stopWg.Done()
      wait.Until(
         func() {
            if !cacher.isStopped() {
               cacher.startCaching(stopCh)
            }
         }, time.Second, stopCh,
      )
   }()
   return cacher
}

再来看看cache.Create方法

// Implements storage.Interface.
func (c *Cacher) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
   // 实际调用了storage的方法。而storage=config.storage. 实际就是之前已经初始化的etcd客户端storage
   return c.storage.Create(ctx, key, obj, out, ttl)
}

总结： 至此，问题2以及问题3想必已经有了答案。