本文基于Kubernetes v1.22.4版本进行源码学习
6、OpenAPI
1）、OpenAPI的作用
OpenAPI是由Swagger发展而来的一个规范，一种形式化描述Restful Service的语言，便于使用者理解和使用一个Service。通过OpenAPI规范可以描述一个服务：
- 提供哪些Restful服务
- 各服务接收的输入以及输出对象格式
- 支持的操作，如get、post等
2）、APIServer Swagger
所在文件：hack/update-openapi-spec.sh，用于生成Swagger文件的脚本
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
OPENAPI_ROOT_DIR="${KUBE_ROOT}/api/openapi-spec"
source "${KUBE_ROOT}/hack/lib/init.sh"
生成的Swagger所在文件：api/openapi-spec/swagger.json
swagger.json文件定义了Kubernetes对外提供的Restful Service，客户端可以依照该规定来向APIServer发HTTP请求。在该文件中，可以看到绝大多数当前版本内建的API Object，并且每个外部版本+API Object的组合拥有swagger中的一套定义。文件内容如下：
- swagger：描述规范的版本，字段类型为string
- info：描述API的基本信息，字段类型为Info Object。Info Object类型包含以下两个必须字段：title表示应用的名称，version表示应用的版本
- paths：描述API的各个端点及支持的操作，字段类型为Paths Object。Paths Object类型由Path Item Object类型构成
- definitions：定义一组被各个接口引用（消费或产生）的对象，类型为Definitions Object
Deployment的Definition部分：
Deployment的Path部分：
3）、Generate OpenAPI Definition
每个API Group目录下的doc.go文件中，使用+k8s:openapi-gen=true注释标识当前package需要生成OpenAPI的内容
// vendor/k8s.io/api/apps/v1/doc.go
// +k8s:openapi-gen=true
package v1 // import "k8s.io/api/apps/v1"
在同package下的types.go文件中，定义该API Group的API Object的结构体
// vendor/k8s.io/api/apps/v1/types.go
type Deployment struct {
metav1.TypeMeta `json:",inline"`
// Standard object's metadata.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
// +optional
metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
// Specification of the desired behavior of the Deployment.
// +optional
Spec DeploymentSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
// Most recently observed status of the Deployment.
// +optional
Status DeploymentStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
}
最终会生成pkg/generated/openapi/zz_generated.openapi.go文件，文件中可以找到当前Kubernetes版本中所有内建的API Object的Open API Definition，并且每个外部版本+API Object的组合会有一个Definition
GetOpenAPIDefinitions()方法返回一个map结构，key是代表外部版本+API Object，value是一个返回对应的Open API Definition的函数
这里的key和swagger.json中的definitionid有一对一映射关系，可以查看vendor/k8s.io/apiserver/pkg/endpoints/openapi/openapi.go文件中的friendlyName()函数
这些Generated Code的作用：为每一个以字符串（就是map的key）标识的API Object，生成其Open API Definition，Definition的主体是一个json schema，该schema定义了这个API Object的json表示中可以有哪些属性、属性类型信息等等
4）、生成OpenAPI Spec
API Server支持通过/openapi/v2这个URI来访问OpenAPI Spec
// cmd/kube-apiserver/app/server.go
func buildGenericConfig(
s *options.ServerRunOptions,
proxyTransport *http.Transport,
) (
genericConfig *genericapiserver.Config,
versionedInformers clientgoinformers.SharedInformerFactory,
serviceResolver aggregatorapiserver.ServiceResolver,
pluginInitializers []admission.PluginInitializer,
admissionPostStartHook genericapiserver.PostStartHookFunc,
storageFactory *serverstorage.DefaultStorageFactory,
lastErr error,
) {
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource()
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
return
}
if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
return
}
if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
return
}
if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil {
return
}
if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil {
return
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
if lastErr = s.Traces.ApplyTo(genericConfig.EgressSelector, genericConfig); lastErr != nil {
return
}
}
// 生成的GetOpenAPIDefinitions会被交给GenericAPIServer的OpenAPIConfig,用于后续生成open api spec
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme))
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig()
storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig
completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd)
if err != nil {
lastErr = err
return
}
storageFactory, lastErr = completedStorageFactoryConfig.New()
if lastErr != nil {
return
}
if genericConfig.EgressSelector != nil {
storageFactory.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) && genericConfig.TracerProvider != nil {
storageFactory.StorageConfig.Transport.TracerProvider = genericConfig.TracerProvider
}
if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil {
return
}
// Use protobufs for self-communication.
// Since not every generic apiserver has to support protobufs, we
// cannot default to it in generic apiserver and need to explicitly
// set it in kube-apiserver.
genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf"
// Disable compression for self-communication, since we are going to be
// on a fast local network
genericConfig.LoopbackClientConfig.DisableCompression = true
kubeClientConfig := genericConfig.LoopbackClientConfig
clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
if err != nil {
lastErr = fmt.Errorf("failed to create real external clientset: %v", err)
return
}
versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
// Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present
if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil {
return
}
genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers)
if err != nil {
lastErr = fmt.Errorf("invalid authorization config: %v", err)
return
}
if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) {
genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
}
lastErr = s.Audit.ApplyTo(genericConfig)
if lastErr != nil {
return
}
admissionConfig := &kubeapiserveradmission.Config{
ExternalInformers: versionedInformers,
LoopbackClientConfig: genericConfig.LoopbackClientConfig,
CloudConfigFile: s.CloudProvider.CloudConfigFile,
}
serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
if err != nil {
lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err)
return
}
err = s.Admission.ApplyTo(
genericConfig,
versionedInformers,
kubeClientConfig,
utilfeature.DefaultFeatureGate,
pluginInitializers...)
if err != nil {
lastErr = fmt.Errorf("failed to initialize admission: %v", err)
return
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnablePriorityAndFairness {
genericConfig.FlowControl, lastErr = BuildPriorityAndFairness(s, clientgoExternalClient, versionedInformers)
}
return
}
在构建GenericAPIServer配置的时候，生成的GetOpenAPIDefinitions会被交给GenericAPIServer的OpenAPIConfig，用于后续生成OpenAPI Spec
现在GenericAPIServer有了OpenAPI的Definition，在GenericAPIServer的Run()之前有个PrepareRun()会进行OpenAPI的配置
// vendor/k8s.io/apiserver/pkg/server/genericapiserver.go
func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
s.delegationTarget.PrepareRun()
// 如果设置了openAPIConfig调用Install方法
if s.openAPIConfig != nil && !s.skipOpenAPIInstallation {
s.OpenAPIVersionedService, s.StaticOpenAPISpec = routes.OpenAPI{
Config: s.openAPIConfig,
}.Install(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
}
s.installHealthz()
s.installLivez()
// as soon as shutdown is initiated, readiness should start failing
readinessStopCh := s.lifecycleSignals.ShutdownInitiated.Signaled()
err := s.addReadyzShutdownCheck(readinessStopCh)
if err != nil {
klog.Errorf("Failed to install readyz shutdown check %s", err)
}
s.installReadyz()
// Register audit backend preShutdownHook.
if s.AuditBackend != nil {
err := s.AddPreShutdownHook("audit-backend", func() error {
s.AuditBackend.Shutdown()
return nil
})
if err != nil {
klog.Errorf("Failed to add pre-shutdown hook for audit-backend %s", err)
}
}
return preparedGenericAPIServer{s}
}
PrepareRun()方法中判断如果设置了openAPIConfig调用Install()方法
// vendor/k8s.io/apiserver/pkg/server/routes/openapi.go
func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) (*handler.OpenAPIService, *spec.Swagger) {
// 1)根据openAPIConfig构建openAPISpec
spec, err := builder.BuildOpenAPISpec(c.RegisteredWebServices(), oa.Config)
if err != nil {
klog.Fatalf("Failed to build open api spec for root: %v", err)
}
spec.Definitions = handler.PruneDefaults(spec.Definitions)
openAPIVersionedService, err := handler.NewOpenAPIService(spec)
if err != nil {
klog.Fatalf("Failed to create OpenAPIService: %v", err)
}
// 2)注册/openapi/v2 uri来访问openAPISpec
err = openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
if err != nil {
klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
}
return openAPIVersionedService, spec
}
Install()方法逻辑如下：
- 代码1)处利用kube-openapi库，生成一个OpenAPI Spec，oa.Config包含了OpenAPIConfig，它是基于生成的GetOpenAPIDefinitions做出来的；生成过程还需要知道系统内API Object的uri，从而生成path，所以这里给了webServices参数
- 代码2)处当用户请求/openapi/v2时，OpenAPI Spec信息会被这个mux返回
7、API Resource的装载
GenericAPIServer提供了对应的方法支持子APIServer把自己支持的Restful Endpoint注入进来：
- InstallLegacyAPIGroup：注册core APIGroup（core APIGroup）下的Resources
- InstallAPIGroups：注册非core APIGroup下的Resources
函数之间调用逻辑如下：
最核心的是registerResourceHandlers()方法，代码如下：
// vendor/k8s.io/apiserver/pkg/endpoints/installer.go
func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, *storageversion.ResourceInfo, error) {
admit := a.group.Admit
// 1)获取被注册的object的group与version,确定是不是subresource
optionsExternalVersion := a.group.GroupVersion
if a.group.OptionsExternalVersion != nil {
optionsExternalVersion = *a.group.OptionsExternalVersion
}
resource, subresource, err := splitSubresource(path)
if err != nil {
return nil, nil, err
}
group, version := a.group.GroupVersion.Group, a.group.GroupVersion.Version
fqKindToRegister, err := GetResourceKind(a.group.GroupVersion, storage, a.group.Typer)
if err != nil {
return nil, nil, err
}
versionedPtr, err := a.group.Creater.New(fqKindToRegister)
if err != nil {
return nil, nil, err
}
defaultVersionedObject := indirectArbitraryPointer(versionedPtr)
kind := fqKindToRegister.Kind
isSubresource := len(subresource) > 0
// 2)确定其区不区分namespace
// If there is a subresource, namespace scoping is defined by the parent resource
namespaceScoped := true
if isSubresource {
parentStorage, ok := a.group.Storage[resource]
if !ok {
return nil, nil, fmt.Errorf("missing parent storage: %q", resource)
}
scoper, ok := parentStorage.(rest.Scoper)
if !ok {
return nil, nil, fmt.Errorf("%q must implement scoper", resource)
}
namespaceScoped = scoper.NamespaceScoped()
} else {
scoper, ok := storage.(rest.Scoper)
if !ok {
return nil, nil, fmt.Errorf("%q must implement scoper", resource)
}
namespaceScoped = scoper.NamespaceScoped()
}
// 3)根据传入的storage对象实现的接口,确定其支持的各种操作(verbs)
// what verbs are supported by the storage, used to know what verbs we support per path
creater, isCreater := storage.(rest.Creater)
namedCreater, isNamedCreater := storage.(rest.NamedCreater)
lister, isLister := storage.(rest.Lister)
getter, isGetter := storage.(rest.Getter)
getterWithOptions, isGetterWithOptions := storage.(rest.GetterWithOptions)
gracefulDeleter, isGracefulDeleter := storage.(rest.GracefulDeleter)
collectionDeleter, isCollectionDeleter := storage.(rest.CollectionDeleter)
updater, isUpdater := storage.(rest.Updater)
patcher, isPatcher := storage.(rest.Patcher)
watcher, isWatcher := storage.(rest.Watcher)
connecter, isConnecter := storage.(rest.Connecter)
storageMeta, isMetadata := storage.(rest.StorageMetadata)
storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider)
gvAcceptor, _ := storage.(rest.GroupVersionAcceptor)
if !isMetadata {
storageMeta = defaultStorageMetadata{}
}
if isNamedCreater {
isCreater = true
}
var resetFields map[fieldpath.APIVersion]*fieldpath.Set
if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
if resetFieldsStrategy, isResetFieldsStrategy := storage.(rest.ResetFieldsStrategy); isResetFieldsStrategy {
resetFields = resetFieldsStrategy.GetResetFields()
}
}
var versionedList interface{}
if isLister {
list := lister.NewList()
listGVKs, _, err := a.group.Typer.ObjectKinds(list)
if err != nil {
return nil, nil, err
}
versionedListPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listGVKs[0].Kind))
if err != nil {
return nil, nil, err
}
versionedList = indirectArbitraryPointer(versionedListPtr)
}
// 4)创建各种ListOptions、CreateOptions、PatchOptions、UpdateOptions以及其他各种options
versionedListOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ListOptions"))
if err != nil {
return nil, nil, err
}
versionedCreateOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("CreateOptions"))
if err != nil {
return nil, nil, err
}
versionedPatchOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("PatchOptions"))
if err != nil {
return nil, nil, err
}
versionedUpdateOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("UpdateOptions"))
if err != nil {
return nil, nil, err
}
var versionedDeleteOptions runtime.Object
var versionedDeleterObject interface{}
deleteReturnsDeletedObject := false
if isGracefulDeleter {
versionedDeleteOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind("DeleteOptions"))
if err != nil {
return nil, nil, err
}
versionedDeleterObject = indirectArbitraryPointer(versionedDeleteOptions)
if mayReturnFullObjectDeleter, ok := storage.(rest.MayReturnFullObjectDeleter); ok {
deleteReturnsDeletedObject = mayReturnFullObjectDeleter.DeleteReturnsDeletedObject()
}
}
versionedStatusPtr, err := a.group.Creater.New(optionsExternalVersion.WithKind("Status"))
if err != nil {
return nil, nil, err
}
versionedStatus := indirectArbitraryPointer(versionedStatusPtr)
var (
getOptions runtime.Object
versionedGetOptions runtime.Object
getOptionsInternalKind schema.GroupVersionKind
getSubpath bool
)
if isGetterWithOptions {
getOptions, getSubpath, _ = getterWithOptions.NewGetOptions()
getOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(getOptions)
if err != nil {
return nil, nil, err
}
getOptionsInternalKind = getOptionsInternalKinds[0]
versionedGetOptions, err = a.group.Creater.New(a.group.GroupVersion.WithKind(getOptionsInternalKind.Kind))
if err != nil {
versionedGetOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(getOptionsInternalKind.Kind))
if err != nil {
return nil, nil, err
}
}
isGetter = true
}
var versionedWatchEvent interface{}
if isWatcher {
versionedWatchEventPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind("WatchEvent"))
if err != nil {
return nil, nil, err
}
versionedWatchEvent = indirectArbitraryPointer(versionedWatchEventPtr)
}
var (
connectOptions runtime.Object
versionedConnectOptions runtime.Object
connectOptionsInternalKind schema.GroupVersionKind
connectSubpath bool
)
if isConnecter {
connectOptions, connectSubpath, _ = connecter.NewConnectOptions()
if connectOptions != nil {
connectOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(connectOptions)
if err != nil {
return nil, nil, err
}
connectOptionsInternalKind = connectOptionsInternalKinds[0]
versionedConnectOptions, err = a.group.Creater.New(a.group.GroupVersion.WithKind(connectOptionsInternalKind.Kind))
if err != nil {
versionedConnectOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(connectOptionsInternalKind.Kind))
if err != nil {
return nil, nil, err
}
}
}
}
allowWatchList := isWatcher && isLister // watching on lists is allowed only for kinds that support both watch and list.
nameParam := ws.PathParameter("name", "name of the "+kind).DataType("string")
pathParam := ws.PathParameter("path", "path to the resource").DataType("string")
params := []*restful.Parameter{}
actions := []action{}
var resourceKind string
kindProvider, ok := storage.(rest.KindProvider)
if ok {
resourceKind = kindProvider.Kind()
} else {
resourceKind = kind
}
tableProvider, isTableProvider := storage.(rest.TableConvertor)
if isLister && !isTableProvider {
// All listers must implement TableProvider
return nil, nil, fmt.Errorf("%q must implement TableConvertor", resource)
}
// 5)生成apiResource,后续会返回
var apiResource metav1.APIResource
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionHash) &&
isStorageVersionProvider &&
storageVersionProvider.StorageVersion() != nil {
versioner := storageVersionProvider.StorageVersion()
gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer)
if err != nil {
return nil, nil, err
}
apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind)
}
// Get the list of actions for the given scope.
// 6)制作actions list,每个resource的每个verb一条记录
switch {
case !namespaceScoped:
// Handle non-namespace scoped resources like nodes.
resourcePath := resource
resourceParams := params
itemPath := resourcePath + "/{name}"
nameParams := append(params, nameParam)
proxyParams := append(nameParams, pathParam)
suffix := ""
if isSubresource {
suffix = "/" + subresource
itemPath = itemPath + suffix
resourcePath = itemPath
resourceParams = nameParams
}
apiResource.Name = path
apiResource.Namespaced = false
apiResource.Kind = resourceKind
namer := handlers.ContextBasedNaming{
SelfLinker: a.group.Linker,
ClusterScoped: true,
SelfLinkPathPrefix: gpath.Join(a.prefix, resource) + "/",
SelfLinkPathSuffix: suffix,
}
// Handler for standard REST verbs (GET, PUT, POST and DELETE).
// Add actions at the resource path: /api/apiVersion/resource
actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)
actions = appendIf(actions, action{"POST", resourcePath, resourceParams, namer, false}, isCreater)
actions = appendIf(actions, action{"DELETECOLLECTION", resourcePath, resourceParams, namer, false}, isCollectionDeleter)
// DEPRECATED in 1.11
actions = appendIf(actions, action{"WATCHLIST", "watch/" + resourcePath, resourceParams, namer, false}, allowWatchList)
// Add actions at the item path: /api/apiVersion/resource/{name}
actions = appendIf(actions, action{"GET", itemPath, nameParams, namer, false}, isGetter)
if getSubpath {
actions = appendIf(actions, action{"GET", itemPath + "/{path:*}", proxyParams, namer, false}, isGetter)
}
actions = appendIf(actions, action{"PUT", itemPath, nameParams, namer, false}, isUpdater)
actions = appendIf(actions, action{"PATCH", itemPath, nameParams, namer, false}, isPatcher)
actions = appendIf(actions, action{"DELETE", itemPath, nameParams, namer, false}, isGracefulDeleter)
// DEPRECATED in 1.11
actions = appendIf(actions, action{"WATCH", "watch/" + itemPath, nameParams, namer, false}, isWatcher)
actions = appendIf(actions, action{"CONNECT", itemPath, nameParams, namer, false}, isConnecter)
actions = appendIf(actions, action{"CONNECT", itemPath + "/{path:*}", proxyParams, namer, false}, isConnecter && connectSubpath)
default:
namespaceParamName := "namespaces"
// Handler for standard REST verbs (GET, PUT, POST and DELETE).
namespaceParam := ws.PathParameter("namespace", "object name and auth scope, such as for teams and projects").DataType("string")
namespacedPath := namespaceParamName + "/{namespace}/" + resource
namespaceParams := []*restful.Parameter{namespaceParam}
resourcePath := namespacedPath
resourceParams := namespaceParams
itemPath := namespacedPath + "/{name}"
nameParams := append(namespaceParams, nameParam)
proxyParams := append(nameParams, pathParam)
itemPathSuffix := ""
if isSubresource {
itemPathSuffix = "/" + subresource
itemPath = itemPath + itemPathSuffix
resourcePath = itemPath
resourceParams = nameParams
}
apiResource.Name = path
apiResource.Namespaced = true
apiResource.Kind = resourceKind
namer := handlers.ContextBasedNaming{
SelfLinker: a.group.Linker,
ClusterScoped: false,
SelfLinkPathPrefix: gpath.Join(a.prefix, namespaceParamName) + "/",
SelfLinkPathSuffix: itemPathSuffix,
}
actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)
actions = appendIf(actions, action{"POST", resourcePath, resourceParams, namer, false}, isCreater)
actions = appendIf(actions, action{"DELETECOLLECTION", resourcePath, resourceParams, namer, false}, isCollectionDeleter)
// DEPRECATED in 1.11
actions = appendIf(actions, action{"WATCHLIST", "watch/" + resourcePath, resourceParams, namer, false}, allowWatchList)
actions = appendIf(actions, action{"GET", itemPath, nameParams, namer, false}, isGetter)
if getSubpath {
actions = appendIf(actions, action{"GET", itemPath + "/{path:*}", proxyParams, namer, false}, isGetter)
}
actions = appendIf(actions, action{"PUT", itemPath, nameParams, namer, false}, isUpdater)
actions = appendIf(actions, action{"PATCH", itemPath, nameParams, namer, false}, isPatcher)
actions = appendIf(actions, action{"DELETE", itemPath, nameParams, namer, false}, isGracefulDeleter)
// DEPRECATED in 1.11
actions = appendIf(actions, action{"WATCH", "watch/" + itemPath, nameParams, namer, false}, isWatcher)
actions = appendIf(actions, action{"CONNECT", itemPath, nameParams, namer, false}, isConnecter)
actions = appendIf(actions, action{"CONNECT", itemPath + "/{path:*}", proxyParams, namer, false}, isConnecter && connectSubpath)
// list or post across namespace.
// For ex: LIST all pods in all namespaces by sending a LIST request at /api/apiVersion/pods.
// TODO: more strongly type whether a resource allows these actions on "all namespaces" (bulk delete)
if !isSubresource {
actions = appendIf(actions, action{"LIST", resource, params, namer, true}, isLister)
// DEPRECATED in 1.11
actions = appendIf(actions, action{"WATCHLIST", "watch/" + resource, params, namer, true}, allowWatchList)
}
}
var resourceInfo *storageversion.ResourceInfo
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) &&
isStorageVersionProvider &&
storageVersionProvider.StorageVersion() != nil {
// 7)决定放入etcd时使用的version,以及从etcd取出时可以转换为的version
versioner := storageVersionProvider.StorageVersion()
encodingGVK, err := getStorageVersionKind(versioner, storage, a.group.Typer)
if err != nil {
return nil, nil, err
}
decodableVersions := []schema.GroupVersion{}
if a.group.ConvertabilityChecker != nil {
decodableVersions = a.group.ConvertabilityChecker.VersionsForGroupKind(fqKindToRegister.GroupKind())
}
// 8)生成resourceInfo,后续会返回
resourceInfo = &storageversion.ResourceInfo{
GroupResource: schema.GroupResource{
Group: a.group.GroupVersion.Group,
Resource: apiResource.Name,
},
EncodingVersion: encodingGVK.GroupVersion().String(),
// We record EquivalentResourceMapper first instead of calculate
// DecodableVersions immediately because API installation must
// be completed first for us to know equivalent APIs
EquivalentResourceMapper: a.group.EquivalentResourceRegistry,
DirectlyDecodableVersions: decodableVersions,
}
}
// Create Routes for the actions.
// TODO: Add status documentation using Returns()
// Errors (see api/errors/errors.go as well as go-restful router):
// http.StatusNotFound, http.StatusMethodNotAllowed,
// http.StatusUnsupportedMediaType, http.StatusNotAcceptable,
// http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden,
// http.StatusRequestTimeout, http.StatusConflict, http.StatusPreconditionFailed,
// http.StatusUnprocessableEntity, http.StatusInternalServerError,
// http.StatusServiceUnavailable
// and api error codes
// Note that if we specify a versioned Status object here, we may need to
// create one for the tests, also
// Success:
// http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent
//
// test/integration/auth_test.go is currently the most comprehensive status code test
for _, s := range a.group.Serializer.SupportedMediaTypes() {
if len(s.MediaTypeSubType) == 0 || len(s.MediaTypeType) == 0 {
return nil, nil, fmt.Errorf("all serializers in the group Serializer must have MediaTypeType and MediaTypeSubType set: %s", s.MediaType)
}
}
// 9)根据Serializer得出支持的mediaTypes,从而设置webservice支持的response属性
mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer)
allMediaTypes := append(mediaTypes, streamMediaTypes...)
ws.Produces(allMediaTypes...)
kubeVerbs := map[string]struct{}{}
// 10)把以上各个环节得到的信息,放入reqScope中
reqScope := handlers.RequestScope{
Serializer: a.group.Serializer,
ParameterCodec: a.group.ParameterCodec,
Creater: a.group.Creater,
Convertor: a.group.Convertor,
Defaulter: a.group.Defaulter,
Typer: a.group.Typer,
UnsafeConvertor: a.group.UnsafeConvertor,
Authorizer: a.group.Authorizer,
EquivalentResourceMapper: a.group.EquivalentResourceRegistry,
// TODO: Check for the interface on storage
TableConvertor: tableProvider,
// TODO: This seems wrong for cross-group subresources. It makes an assumption that a subresource and its parent are in the same group version. Revisit this.
Resource: a.group.GroupVersion.WithResource(resource),
Subresource: subresource,
Kind: fqKindToRegister,
AcceptsGroupVersionDelegate: gvAcceptor,
HubGroupVersion: schema.GroupVersion{Group: fqKindToRegister.Group, Version: runtime.APIVersionInternal},
MetaGroupVersion: metav1.SchemeGroupVersion,
MaxRequestBodyBytes: a.group.MaxRequestBodyBytes,
}
if a.group.MetaGroupVersion != nil {
reqScope.MetaGroupVersion = *a.group.MetaGroupVersion
}
// 11)根据情况计算reqScope的FieldManager属性
if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
reqScope.FieldManager, err = fieldmanager.NewDefaultFieldManager(
a.group.TypeConverter,
a.group.UnsafeConvertor,
a.group.Defaulter,
a.group.Creater,
fqKindToRegister,
reqScope.HubGroupVersion,
subresource,
resetFields,
)
if err != nil {
return nil, nil, fmt.Errorf("failed to create field manager: %v", err)
}
}
// 12)逐个处理actions list中的action,基于reqScope等信息,为他们生成route并注册到webservice中去
for _, action := range actions {
producedObject := storageMeta.ProducesObject(action.Verb)
if producedObject == nil {
producedObject = defaultVersionedObject
}
reqScope.Namer = action.Namer
requestScope := "cluster"
var namespaced string
var operationSuffix string
if apiResource.Namespaced {
requestScope = "namespace"
namespaced = "Namespaced"
}
if strings.HasSuffix(action.Path, "/{path:*}") {
requestScope = "resource"
operationSuffix = operationSuffix + "WithPath"
}
if strings.Index(action.Path, "/{name}") != -1 || action.Verb == "POST" {
requestScope = "resource"
}
if action.AllNamespaces {
requestScope = "cluster"
operationSuffix = operationSuffix + "ForAllNamespaces"
namespaced = ""
}
if kubeVerb, found := toDiscoveryKubeVerb[action.Verb]; found {
if len(kubeVerb) != 0 {
kubeVerbs[kubeVerb] = struct{}{}
}
} else {
return nil, nil, fmt.Errorf("unknown action verb for discovery: %s", action.Verb)
}
routes := []*restful.RouteBuilder{}
// If there is a subresource, kind should be the parent's kind.
if isSubresource {
parentStorage, ok := a.group.Storage[resource]
if !ok {
return nil, nil, fmt.Errorf("missing parent storage: %q", resource)
}
fqParentKind, err := GetResourceKind(a.group.GroupVersion, parentStorage, a.group.Typer)
if err != nil {
return nil, nil, err
}
kind = fqParentKind.Kind
}
verbOverrider, needOverride := storage.(StorageMetricsOverride)
// accumulate endpoint-level warnings
var (
warnings []string
deprecated bool
removedRelease string
)
{
versionedPtrWithGVK := versionedPtr.DeepCopyObject()
versionedPtrWithGVK.GetObjectKind().SetGroupVersionKind(fqKindToRegister)
currentMajor, currentMinor, _ := deprecation.MajorMinor(versioninfo.Get())
deprecated = deprecation.IsDeprecated(versionedPtrWithGVK, currentMajor, currentMinor)
if deprecated {
removedRelease = deprecation.RemovedRelease(versionedPtrWithGVK)
warnings = append(warnings, deprecation.WarningMessage(versionedPtrWithGVK))
}
}
switch action.Verb {
case "GET": // Get a resource.
var handler restful.RouteFunction
if isGetterWithOptions {
handler = restfulGetResourceWithOptions(getterWithOptions, reqScope, isSubresource)
} else {
handler = restfulGetResource(getter, reqScope)
}
if needOverride {
// need change the reported verb
handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
} else {
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
doc := "read the specified " + kind
if isSubresource {
doc = "read " + subresource + " of the specified " + kind
}
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("read"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Returns(http.StatusOK, "OK", producedObject).
Writes(producedObject)
if isGetterWithOptions {
if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
return nil, nil, err
}
}
addParams(route, action.Params)
routes = append(routes, route)
case "LIST": // List all resources of a kind.
doc := "list objects of kind " + kind
if isSubresource {
doc = "list " + subresource + " of objects of kind " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("list"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), allMediaTypes...)...).
Returns(http.StatusOK, "OK", versionedList).
Writes(versionedList)
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, nil, err
}
switch {
case isLister && isWatcher:
doc := "list or watch objects of kind " + kind
if isSubresource {
doc = "list or watch " + subresource + " of objects of kind " + kind
}
route.Doc(doc)
case isWatcher:
doc := "watch objects of kind " + kind
if isSubresource {
doc = "watch " + subresource + "of objects of kind " + kind
}
route.Doc(doc)
}
addParams(route, action.Params)
routes = append(routes, route)
case "PUT": // Update a resource.
doc := "replace the specified " + kind
if isSubresource {
doc = "replace " + subresource + " of the specified " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulUpdateResource(updater, reqScope, admit))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.PUT(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("replace"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Returns(http.StatusOK, "OK", producedObject).
// TODO: in some cases, the API may return a v1.Status instead of the versioned object
// but currently go-restful can't handle multiple different objects being returned.
Returns(http.StatusCreated, "Created", producedObject).
Reads(defaultVersionedObject).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
case "PATCH": // Partially update a resource
doc := "partially update the specified " + kind
if isSubresource {
doc = "partially update " + subresource + " of the specified " + kind
}
supportedTypes := []string{
string(types.JSONPatchType),
string(types.MergePatchType),
string(types.StrategicMergePatchType),
}
if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
supportedTypes = append(supportedTypes, string(types.ApplyPatchType))
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.PATCH(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Consumes(supportedTypes...).
Operation("patch"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Returns(http.StatusOK, "OK", producedObject).
// Patch can return 201 when a server side apply is requested
Returns(http.StatusCreated, "Created", producedObject).
Reads(metav1.Patch{}).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
case "POST": // Create a resource.
var handler restful.RouteFunction
if isNamedCreater {
handler = restfulCreateNamedResource(namedCreater, reqScope, admit)
} else {
handler = restfulCreateResource(creater, reqScope, admit)
}
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
handler = utilwarning.AddWarningsHandler(handler, warnings)
article := GetArticleForNoun(kind, " ")
doc := "create" + article + kind
if isSubresource {
doc = "create " + subresource + " of" + article + kind
}
route := ws.POST(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("create"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Returns(http.StatusOK, "OK", producedObject).
// TODO: in some cases, the API may return a v1.Status instead of the versioned object
// but currently go-restful can't handle multiple different objects being returned.
Returns(http.StatusCreated, "Created", producedObject).
Returns(http.StatusAccepted, "Accepted", producedObject).
Reads(defaultVersionedObject).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
case "DELETE": // Delete a resource.
article := GetArticleForNoun(kind, " ")
doc := "delete" + article + kind
if isSubresource {
doc = "delete " + subresource + " of" + article + kind
}
deleteReturnType := versionedStatus
if deleteReturnsDeletedObject {
deleteReturnType = producedObject
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.DELETE(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("delete"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Writes(deleteReturnType).
Returns(http.StatusOK, "OK", deleteReturnType).
Returns(http.StatusAccepted, "Accepted", deleteReturnType)
if isGracefulDeleter {
route.Reads(versionedDeleterObject)
route.ParameterNamed("body").Required(false)
if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
return nil, nil, err
}
}
addParams(route, action.Params)
routes = append(routes, route)
case "DELETECOLLECTION":
doc := "delete collection of " + kind
if isSubresource {
doc = "delete collection of " + subresource + " of a " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.DELETE(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("deletecollection"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Writes(versionedStatus).
Returns(http.StatusOK, "OK", versionedStatus)
if isCollectionDeleter {
route.Reads(versionedDeleterObject)
route.ParameterNamed("body").Required(false)
if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
return nil, nil, err
}
}
if err := AddObjectParams(ws, route, versionedListOptions, "watch", "allowWatchBookmarks"); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
// deprecated in 1.11
case "WATCH": // Watch a resource.
doc := "watch changes to an object of kind " + kind
if isSubresource {
doc = "watch changes to " + subresource + " of an object of kind " + kind
}
doc += ". deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter."
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("watch"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(allMediaTypes...).
Returns(http.StatusOK, "OK", versionedWatchEvent).
Writes(versionedWatchEvent)
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
// deprecated in 1.11
case "WATCHLIST": // Watch all resources of a kind.
doc := "watch individual changes to a list of " + kind
if isSubresource {
doc = "watch individual changes to a list of " + subresource + " of " + kind
}
doc += ". deprecated: use the 'watch' parameter with a list operation instead."
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
Operation("watch"+namespaced+kind+strings.Title(subresource)+"List"+operationSuffix).
Produces(allMediaTypes...).
Returns(http.StatusOK, "OK", versionedWatchEvent).
Writes(versionedWatchEvent)
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
case "CONNECT":
for _, method := range connecter.ConnectMethods() {
connectProducedObject := storageMeta.ProducesObject(method)
if connectProducedObject == nil {
connectProducedObject = "string"
}
doc := "connect " + method + " requests to " + kind
if isSubresource {
doc = "connect " + method + " requests to " + subresource + " of " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulConnectResource(connecter, reqScope, admit, path, isSubresource))
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.Method(method).Path(action.Path).
To(handler).
Doc(doc).
Operation("connect" + strings.Title(strings.ToLower(method)) + namespaced + kind + strings.Title(subresource) + operationSuffix).
Produces("*/*").
Consumes("*/*").
Writes(connectProducedObject)
if versionedConnectOptions != nil {
if err := AddObjectParams(ws, route, versionedConnectOptions); err != nil {
return nil, nil, err
}
}
addParams(route, action.Params)
routes = append(routes, route)
// transform ConnectMethods to kube verbs
if kubeVerb, found := toDiscoveryKubeVerb[method]; found {
if len(kubeVerb) != 0 {
kubeVerbs[kubeVerb] = struct{}{}
}
}
}
default:
return nil, nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)
}
for _, route := range routes {
route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{
Group: reqScope.Kind.Group,
Version: reqScope.Kind.Version,
Kind: reqScope.Kind.Kind,
})
route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb))
ws.Route(route)
}
// Note: update GetAuthorizerAttributes() when adding a custom handler.
}
// 13)更新apiResource,后续会返回
apiResource.Verbs = make([]string, 0, len(kubeVerbs))
for kubeVerb := range kubeVerbs {
apiResource.Verbs = append(apiResource.Verbs, kubeVerb)
}
sort.Strings(apiResource.Verbs)
if shortNamesProvider, ok := storage.(rest.ShortNamesProvider); ok {
apiResource.ShortNames = shortNamesProvider.ShortNames()
}
if categoriesProvider, ok := storage.(rest.CategoriesProvider); ok {
apiResource.Categories = categoriesProvider.Categories()
}
if gvkProvider, ok := storage.(rest.GroupVersionKindProvider); ok {
gvk := gvkProvider.GroupVersionKind(a.group.GroupVersion)
apiResource.Group = gvk.Group
apiResource.Version = gvk.Version
apiResource.Kind = gvk.Kind
}
// Record the existence of the GVR and the corresponding GVK
a.group.EquivalentResourceRegistry.RegisterKindFor(reqScope.Resource, reqScope.Subresource, fqKindToRegister)
return &apiResource, resourceInfo, nil
}
registerResourceHandlers()方法主要逻辑如下：
- 获取被注册的object的group与version，确定是不是subresource
- 确定其区不区分namespace
- 根据传入的storage对象实现的接口，确定其支持的各种操作（verbs）
- 创建各种ListOptions、CreateOptions、PatchOptions、UpdateOptions以及其他各种options
- 生成apiResource，后续会返回
- 制作actions list，每个resource的每个verb一条记录
- 决定放入etcd时使用的version，以及从etcd取出时可以转换为的version
- 生成resourceInfo，后续会返回
- 根据Serializer得出支持的mediaTypes，从而设置webservice支持的response属性
- 把以上各个环节得到的信息，放入reqScope中
- 根据情况计算reqScope的FieldManager属性
- 逐个处理actions list中的action，基于reqScope等信息，为他们生成route并注册到webservice中去
- 更新apiResource，后续会返回
8、HTTP Server具体是怎么跑起来的？
1）、HTTP Server的启动
函数之间调用逻辑如下：
2）、Server生命周期状态的流转
vendor/k8s.io/apiserver/pkg/server/genericapiserver.go中的Run()方法利用channel机制编织出server生命周期中的状态流转，代码如下：
// vendor/k8s.io/apiserver/pkg/server/genericapiserver.go
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
// 编织出server生命周期中的状态流转(利用channel机制)
delayedStopCh := s.lifecycleSignals.AfterShutdownDelayDuration
shutdownInitiatedCh := s.lifecycleSignals.ShutdownInitiated
go func() {
defer delayedStopCh.Signal()
defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", delayedStopCh.Name())
<-stopCh
// As soon as shutdown is initiated, /readyz should start returning failure.
// This gives the load balancer a window defined by ShutdownDelayDuration to detect that /readyz is red
// and stop sending traffic to this server.
shutdownInitiatedCh.Signal()
klog.V(1).InfoS("[graceful-termination] shutdown event", "name", shutdownInitiatedCh.Name())
time.Sleep(s.ShutdownDelayDuration)
}()
// 调用s.NonBlockingRun()方法
// close socket after delayed stopCh
stoppedCh, listenerStoppedCh, err := s.NonBlockingRun(delayedStopCh.Signaled())
if err != nil {
return err
}
httpServerStoppedListeningCh := s.lifecycleSignals.HTTPServerStoppedListening
go func() {
<-listenerStoppedCh
httpServerStoppedListeningCh.Signal()
klog.V(1).InfoS("[graceful-termination] shutdown event", "name", httpServerStoppedListeningCh.Name())
}()
drainedCh := s.lifecycleSignals.InFlightRequestsDrained
go func() {
defer drainedCh.Signal()
defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", drainedCh.Name())
// wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called).
<-delayedStopCh.Signaled()
// Wait for all requests to finish, which are bounded by the RequestTimeout variable.
s.HandlerChainWaitGroup.Wait()
}()
klog.V(1).Info("[graceful-termination] waiting for shutdown to be initiated")
<-stopCh
// 关闭开始前调用pre shutdown hooks
// run shutdown hooks directly. This includes deregistering from the kubernetes endpoint in case of kube-apiserver.
err = s.RunPreShutdownHooks()
if err != nil {
return err
}
klog.V(1).Info("[graceful-termination] RunPreShutdownHooks has completed")
// Wait for all requests in flight to drain, bounded by the RequestTimeout variable.
<-drainedCh.Signaled()
// wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished.
<-stoppedCh
klog.V(1).Info("[graceful-termination] apiserver is exiting")
return nil
}
lifecycleSignals结构体中定义了Server生命周期的事件：
// vendor/k8s.io/apiserver/pkg/server/lifecycle_signals.go
type lifecycleSignals struct {
// ShutdownInitiated event is signaled when an apiserver shutdown has been initiated.
// It is signaled when the `stopCh` provided by the main goroutine
// receives a KILL signal and is closed as a consequence.
ShutdownInitiated lifecycleSignal
// AfterShutdownDelayDuration event is signaled as soon as ShutdownDelayDuration
// has elapsed since the ShutdownInitiated event.
// ShutdownDelayDuration allows the apiserver to delay shutdown for some time.
AfterShutdownDelayDuration lifecycleSignal
// InFlightRequestsDrained event is signaled when the existing requests
// in flight have completed. This is used as signal to shut down the audit backends
InFlightRequestsDrained lifecycleSignal
// HTTPServerStoppedListening termination event is signaled when the
// HTTP Server has stopped listening to the underlying socket.
HTTPServerStoppedListening lifecycleSignal
// HasBeenReady is signaled when the readyz endpoint succeeds for the first time.
HasBeenReady lifecycleSignal
}
Server生命周期状态的流转如下图：
3）、preparedGenericAPIServer.NonBlockingRun
函数之间调用逻辑如下：
参考：文章来源：https://www.toymoban.com/news/detail-680945.html
Kubernetes源码开发之旅三：API Server源码剖析文章来源地址https://www.toymoban.com/news/detail-680945.html
到了这里，关于Kubernetes API Server源码学习（二）：OpenAPI、API Resource的装载、HTTP Server具体是怎么跑起来的？的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
