rancher开发

发表于 | 分类于 | 阅读次数

rancher

项目结构

./
| -- pkg
|    -- api  // api的初始化、导入
|    -- apis // api(crd)的定义
|    -- client/generated // norman生成
|    -- codegen // 代码生成程序
|    -- controllers // 控制器的业务代码逻辑
|    -- generated // wrangler生成
|    -- schemas // api的Schema描述配置

CRD 定义

k8s的资源：主要分core族（/api/{version}/前缀）、apis族（/apis/{group}/{version}前缀），每种资源属于某个族下面的组(core族不区分组，只有版本)，并且可以有不同的版本。

https://github.com/rancher/rancher/blob/release/v2.6/pkg/apis/management.cattle.io/v3/cluster_types.go

以pkg/apis/management.cattle.io/v3这个包为例。rancher的代码生成分两部分，

  • crd的新增和导入，包括api的schema(请求参数/访问路由/数据校验/…)定义，

  • controller基础代码。两者依赖normal和wrangler。

image-20220528102613901

crd定义crd_types.go

资源级别

CRD 具有以下 2 个字段以使其成为集群级资源

1
2
types.Namespaced
ClusterName string `json:"clusterName" norman:"type=reference[cluster]"

CRD 具有这两个字段以使其成为项目级资源

1
2
types.Namespaced
ProjectName string `json:"projectName" norman:"type=reference[project]"

schema 定义 api response

pkg/schemas/*

https://github.com/rancher/rancher/blob/release/v2.6/pkg/schemas/management.cattle.io/v3/schema.go

schema

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
func clusterTypes(schemas *types.Schemas) *types.Schemas {
	return schemas.
    // 该函数不会生成Schema，而是将Schemas对象中创建的Mapper对象保持为资源类型Key。然后，当通过 Schemas 对象创建对应资源类型的 Schema 对象时，Mapper 被设置为 Schema
  //指定版本（第一个参数）和资源（第二个参数），并创建对应的 Mapper。Mapper 可以在第三个参数后指定多个资源转换规则。
		AddMapperForType(&Version, v3.Cluster{},
			&m.Embed{Field: "status"},
			mapper.NewDropFromSchema("genericEngineConfig"),
			mapper.NewDropFromSchema("googleKubernetesEngineConfig"),
			mapper.NewDropFromSchema("azureKubernetesServiceConfig"),
			mapper.NewDropFromSchema("amazonElasticContainerServiceConfig"),
			m.DisplayName{},
		).
		AddMapperForType(&Version, v3.ClusterStatus{},
			m.Drop{Field: "serviceAccountToken"},
		).
  // 将 Kubernetes 上 ClusterRegistrationToken 资源的“status”字段转换为 API 资源时，展开“status”字段中的子字段
		AddMapperForType(&Version, v3.ClusterRegistrationToken{},
			&m.Embed{Field: "status"},
		).
		AddMapperForType(&Version, rketypes.RancherKubernetesEngineConfig{},
//将 Kubernetes 上的 RancherKubernetesEngineConfig 资源转换为 API 资源时，删除其“systemImages”字段                     
			m.Drop{Field: "systemImages"},
		).
		...
		MustImport(&Version, v3.RestoreFromEtcdBackupInput{}).
  // 指定版本（第一个参数）和资源（第二个参数），并创建对应资源类型的 Schema。
		MustImport(&Version, v3.SaveAsTemplateInput{}).
		MustImport(&Version, v3.SaveAsTemplateOutput{}).
		AddMapperForType(&Version, v1.EnvVar{},
     // 将 Kubernetes 上的 EnvVar 资源的“envVar”字段转换为 API 资源时的“agentEnvVar”                     
			&m.Move{
				From: "envVar",
				To:   "agentEnvVar",
			}).
  // 指定版本（第一个参数）和资源（第二个参数），并创建对应资源类型的 Schema。创建 Schema 后，该函数可以通过第三个参数的函数指定要为 Schema 执行的附加逻辑。此函数在创建 Schema 后以 Schema 作为参数执行。
		MustImportAndCustomize(&Version, rketypes.ETCDService{}, func(schema *types.Schema) {
			schema.MustCustomizeField("extraArgs", func(field types.Field) types.Field {
				field.Default = map[string]interface{}{
					"election-timeout":   "5000",
					"heartbeat-interval": "500"}
				return field
			})
		}).
		MustImportAndCustomize(&Version, v3.Cluster{}, func(schema *types.Schema) {
			schema.MustCustomizeField("name", func(field types.Field) types.Field {
				field.Type = "dnsLabel"
				field.Nullable = true
				field.Required = false
				return field
			})
			schema.ResourceActions[v3.ClusterActionGenerateKubeconfig] = types.Action{
				Output: "generateKubeConfigOutput",
			}
		.....
			schema.ResourceActions[v3.ClusterActionSaveAsTemplate] = types.Action{
				Input:  "saveAsTemplateInput",
				Output: "saveAsTemplateOutput",
			}
		})
}

定义好以后初始化

1
2
3
4
5
	Schemas = factory.Schemas(&Version).
		Init(nativeNodeTypes).
		Init(nodeTypes).
		Init(authzTypes).
		Init(clusterTypes)

生成代码

包括crd新增导入，api定义，controller基础代码

1
1
go run pkg/codegen/main.go

https://github.com/rancher/rancher/blob/release/v2.6/pkg/api/norman/server/managementstored/setup.go

定义shema把串联起来

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
func Clusters(ctx context.Context, schemas *types.Schemas, managementContext *config.ScaledContext, clusterManager *clustermanager.Manager, k8sProxy http.Handler) {
	schema := schemas.Schema(&managementschema.Version, client.ClusterType)
	clusterFormatter := ccluster.NewFormatter(schemas, managementContext)
	schema.Formatter = clusterFormatter.Formatter
	schema.CollectionFormatter = clusterFormatter.CollectionFormatter
	clusterStore := cluster.GetClusterStore(schema, managementContext, clusterManager, k8sProxy)
	schema.Store = clusterStore

	handler := ccluster.ActionHandler{
		NodepoolGetter:                managementContext.Management,
		NodeLister:                    managementContext.Management.Nodes("").Controller().Lister(),
		ClusterClient:                 managementContext.Management.Clusters(""),
		CatalogManager:                managementContext.CatalogManager,
		UserMgr:                       managementContext.UserManager,
		ClusterManager:                clusterManager,
		NodeTemplateGetter:            managementContext.Management,
		BackupClient:                  managementContext.Management.EtcdBackups(""),
		ClusterTemplateClient:         managementContext.Management.ClusterTemplates(""),
		ClusterTemplateRevisionClient: managementContext.Management.ClusterTemplateRevisions(""),
		SubjectAccessReviewClient:     managementContext.K8sClient.AuthorizationV1().SubjectAccessReviews(),
		TokenClient:                   managementContext.Management.Tokens(""),
		Auth:                          requests.NewAuthenticator(ctx, clusterrouter.GetClusterID, managementContext),
	}

	clusterValidator := ccluster.Validator{
		ClusterClient:                 managementContext.Management.Clusters(""),
		ClusterLister:                 managementContext.Management.Clusters("").Controller().Lister(),
		ClusterTemplateLister:         managementContext.Management.ClusterTemplates("").Controller().Lister(),
		ClusterTemplateRevisionLister: managementContext.Management.ClusterTemplateRevisions("").Controller().Lister(),
		Users:                         managementContext.Management.Users(""),
		GrbLister:                     managementContext.Management.GlobalRoleBindings("").Controller().Lister(),
		GrLister:                      managementContext.Management.GlobalRoles("").Controller().Lister(),
	}

	handler.ClusterScanClient = managementContext.Management.ClusterScans("")
	handler.CatalogTemplateVersionLister = managementContext.Management.CatalogTemplateVersions("").Controller().Lister()
	handler.CisConfigClient = managementContext.Management.CisConfigs("")
	handler.CisConfigLister = managementContext.Management.CisConfigs("").Controller().Lister()
	handler.CisBenchmarkVersionClient = managementContext.Management.CisBenchmarkVersions("")
	handler.CisBenchmarkVersionLister = managementContext.Management.CisBenchmarkVersions("").Controller().Lister()

	clusterValidator.CisConfigClient = managementContext.Management.CisConfigs(namespace.GlobalNamespace)
	clusterValidator.CisConfigLister = managementContext.Management.CisConfigs(namespace.GlobalNamespace).Controller().Lister()
	clusterValidator.CisBenchmarkVersionClient = managementContext.Management.CisBenchmarkVersions(namespace.GlobalNamespace)
	clusterValidator.CisBenchmarkVersionLister = managementContext.Management.CisBenchmarkVersions(namespace.GlobalNamespace).Controller().Lister()

	schema.ActionHandler = handler.ClusterActionHandler
	schema.Validator = clusterValidator.Validator
}

创建crd

1
2
3
4
5
6
factory.BatchCreateCRDs(ctx, config.ManagementStorageContext, scheme.Scheme, schemas, &managementschema.Version,
  client.SampleCrontabType,
  client.DplmType,
)

Dplms(schemas, apiContext)

image-20220528105956518

validators，stores，formatters用于按序处理请求

validators

API 请求转发到 Kubernetes 之前检查其输入，那么您可能需要使用验证器。验证器的另一个用途是阻止一种请求。值得一提的是，验证器仅适用于 Post 和 Put 请求。

包含验证器的文件位于 pkg/api/norman/customization/ 文件夹中在

https://github.com/rancher/rancher/blob/release/v2.6/pkg/api/norman/customization/cluster/validator.go

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
func Validator(request *types.APIContext, schema *types.Schema, data map[string]interface{}) error {
	if pathURL, _ := data["url"].(string); pathURL != "" {
		if err := validateURL(pathURL); err != nil {
			return err
		}
		if u, err := url.Parse(pathURL); err == nil {
			u.Scheme = strings.ToLower(u.Scheme) // git commands are case-sensitive
			data["url"] = u.String()
		}
	} else if request.Method == http.MethodPost {
		return httperror.NewAPIError(httperror.MissingRequired, "Catalog URL not specified")
	}
	//
	if helmVersion, ok := data["helmVersion"]; ok {
		toLowerHelmVersion := strings.ToLower(helmVersion.(string))
		// determine if user is setting helmVersion to helm 3 and validate to help user set value correctly
		if strings.Contains(toLowerHelmVersion, "v3") && !common.IsHelm3(toLowerHelmVersion) {
			return httperror.NewAPIError(httperror.InvalidBodyContent, "Invalid helm 3 version")
		}
	}
	return nil
}

func validateURL(pathURL string) error {
	if controlChars.FindStringIndex(pathURL) != nil || controlEncoded.FindStringIndex(pathURL) != nil {
		return httperror.NewAPIError(httperror.InvalidBodyContent, "Invalid characters in catalog URL")
	}
	return nil
}

这是个校验参数的例子

关于必填项norman:”required”,一定要配合json的omitempty使用，不然就会越过校验，noreman只是检查是否存在key，不关心value是否为空

1
2
3
4
5
6
7
8
9
type CatalogSpec struct {
	Description string `json:"description"`
	URL         string `json:"url,omitempty" norman:"required"`
	Branch      string `json:"branch,omitempty"`
	CatalogKind string `json:"catalogKind,omitempty"`
	Username    string `json:"username,omitempty"`
	Password    string `json:"password,omitempty" norman:"type=password"`
	HelmVersion string `json:"helmVersion,omitempty" norman:"noupdate"`
}

Formatters

用于改变api response的schema输出。和validators的创建和分配差不多。是唯一可以编辑link的地方

Action Handlers

处理除了create, update, delete, get, watch的操作的api

Store

Stores can transform and filter requests, similar to formatters and validators. Unlike validators, stores can be used for Delete, Watch, List, and ByID, in addition to Create and Update.

controller

这里以cluster为例。

Contexts分类

  • one ScaledContext per management server (3)
  • one ManagementContext per management server master (1)
  • one UserContext per downstream cluster. When referring to contexts a ‘user’ means a single downstream cluster.

放在pkg/controllers

  • management controllers 用 ManagementContext and handles actions 管理master server，例如全局变化
  • user controllers use the UserContext and handle actions 管理下游集群，

AddHandler vs AddClusterScopedHandler vs AddLifecycle

AddHandler

为您的控制器提供其正在侦听的资源的所有实例的更新

AddClusterScopedHandler

更新本集群内所有实例

AddLifecycle

比AddHandler复杂, 按照不同事件区分Create/Update/Delete方法更新

V1

定义shema

image-20220619214358296

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

type Demo struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`
	Spec              DemoSpec   `json:"spec"`
	Status            DemoStatus `json:"status,omitempty"`
}

type DemoSpec struct {
	Label string `json:"label,omitempty"`
}

type DemoStatus struct {
	State         string `json:"state,omitempty"`
	Transitioning bool   `json:"transitioning,omitempty"`
	Error         bool   `json:"error,omitempty"`
}

生成代码

运行

1
1
go run pkg/codegen/main.go

生成pkg/apis/ui.cattle.io/zz_*.go，pkg/generated/controllers/ui.cattle.io/interface.go 和对应的controller

绑定crd

pkg/crds/dashboard/crds.go

1
2
3
4
5
6
7
8
9
10
11
12
func List(cfg *rest.Config) (_ []crd.CRD, err error) {
	result := []crd.CRD{
	.....
		newCRD(&uiv1.Demo{}, func(c crd.CRD) crd.CRD {
			c.Status = false
			c.NonNamespace = true
			c.GVK.Kind = "Demo"
			c.GVK.Group = "ui.cattle.io"
			c.GVK.Version = "v1"
			return c
		})
}

设置无create crd的v1

pkg/codegen/main.go

image-20220619222922690

创建v1/xx_types

image-20220619223245122

然后其他步骤和上面一样

展示

create功能的v1 crd

image-20220619221040583

image-20220619221140353

启用

选用go version 1.17,使用高版本codegen会报错

调试运行需要安装集群和docker，环境配置。

1
2
3
4
export CATTLE_DEV_MODE=30
export CATTLE_SYSTEM_CHART_DEFAULT_BRANCH=dev-v2.6
export KUBECONFIG=~/.kube/config
go run ./main.go --add-local=true

观察启动日志，可看到新定义的crd导入。所有api的浏览链接 https://localhost:8443/v3 OR https://localhost:8443/v1

image-20220528124542059

一个比较麻烦的地方，第一次启动需要翻墙因为默认从官网获取UI，第一次启动后可以在全局设置修改UI获取方式改成local

image-20220528125823241

然后从对应版本的rancher镜像，docker cp出对应的UI文件放在本地。

https://github.com/rancher/rancher/blob/release/v2.6/pkg/settings/setting.go

1
2
3
	UIPath                              = NewSetting("ui-path", "/usr/local/share/rancher/ui")
....
	UIDashboardPath                     = NewSetting("ui-dashboard-path", "/usr/local/share/rancher/ui-dashboard")

目录在这里有两个目录

building

参考这个https://rancher.com/docs/rancher/v2.6/en/contributing/ ，最好在翻墙情况下完成

1
2
3
4
5
# 主要是利用doker,go build 出二进制,然后docker cp到项目bin目录
make build

# 利用bin目录结果，打包镜像,所以要先build
make package

validate 主要是静态检查，test就是跑测试的

package时候要把.dockerignore去掉，这样才会引用到build的结果

1
./bin