【kubernetes/k8s源码分析】 rook operator cluster controller源码分析之三
這篇文章分析osd啓動流程
存储落地操作的节点叫做OSD（Object Storage Device），基于它工作的RADOS、PG等模块
启动命令：
ceph-osd --foreground --id 1 --conf /var/lib/rook/osd1/rook-ceph.config --osd-data /var/lib/rook/osd1 --keyring /var/lib/rook/osd1/keyring --cluster rook-ceph --osd-uuid ec6b67a1-21b2-4acc-9228-c53aad0811c3
config配置如下：
storage: # cluster level storage configuration and selection useAllNodes: true useAllDevices: false deviceFilter: location: config: # The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories. # Set the storeType explicitly only if it is required not to use the default. storeType: bluestore metadataDevice: "sdb" # specify a non-rotational storage so ceph-volume will use it as block db device of bluestore. databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB journalSizeMB: "1024" # uncomment if the disks are 20 GB or smaller # osdsPerDevice: "1" # this value can be overridden at the node or device level # encryptedDevice: "true" # the default value for this option is "false"
Start函數
如果设置了useAllNodes为true，则使用所有节点，"kubernetes.io/hostname": "master-node"
if c.DesiredStorage.UseAllNodes {
// Get the list of all nodes in the cluster. The placement settings will be applied below.
hostnameMap, err := k8sutil.GetNodeHostNames(c.context.Clientset)
if err != nil {
logger.Warningf("failed to get node hostnames: %v", err)
return err
}
c.DesiredStorage.Nodes = nil
for _, hostname := range hostnameMap {
storageNode := rookalpha.Node{
Name: hostname,
}
c.DesiredStorage.Nodes = append(c.DesiredStorage.Nodes, storageNode)
}
logger.Debugf("storage nodes: %+v", c.DesiredStorage.Nodes)
}GetValidNodes函数则根据nodes中是否定义创建osd内容
// generally speaking, this finds nodes which are capable of running new osds
validNodes := k8sutil.GetValidNodes(c.DesiredStorage.Nodes, c.context.Clientset, c.placement)
// no valid node is ready to run an osd
if len(validNodes) == 0 {
logger.Warningf("no valid node available to run an osd in namespace %s. "+
"Rook will not create any new OSD nodes and will skip checking for removed nodes since "+
"removing all OSD nodes without destroying the Rook cluster is unlikely to be intentional", c.Namespace)
return nil
}
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.DesiredStorage.Nodes))
c.ValidStorage.Nodes = validNodes
1 ListDevices調用client-go獲得configmap得到設備信息
allNodeDevices, err := discover.ListDevices(c.context, rookSystemNS, "" /* all nodes */)
if err != nil {
logger.Warningf("failed to get storage nodes from namespace %s: %v", rookSystemNS, err)
return err
}2 startProvisioning函數
如果未設置dataDirHostPath就不處理了
if len(c.dataDirHostPath) == 0 {
logger.Warningf("skipping osd provisioning where no dataDirHostPath is set")
return
}2.1 获取config配置，同上格式
storage: # cluster level storage configuration and selection
useAllNodes: false
useAllDevices: false
deviceFilter: "/dev/sda"
location:
config:
# The default and recommended storeType is dynamically set to bluestore for devices and filestore for directories.
# Set the storeType explicitly only if it is required not to use the default.
storeType: bluestore
# metadataDevice: "md0" # specify a non-rotational storage so ceph-volume will use it as block db device of bluestore.
# databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB
# journalSizeMB: "1024" # uncomment if the disks are 20 GB or smaller
osdsPerDevice: "1" # this value can be overridden at the node or device level
# encryptedDevice: "true" # the default value for this option is "false"
func ToStoreConfig(config map[string]string) StoreConfig {
storeConfig := StoreConfig{}
for k, v := range config {
switch k {
case StoreTypeKey:
storeConfig.StoreType = v
case WalSizeMBKey:
storeConfig.WalSizeMB = convertToIntIgnoreErr(v)
case DatabaseSizeMBKey:
storeConfig.DatabaseSizeMB = convertToIntIgnoreErr(v)
case JournalSizeMBKey:
storeConfig.JournalSizeMB = convertToIntIgnoreErr(v)
case OSDsPerDeviceKey:
storeConfig.OSDsPerDevice = convertToIntIgnoreErr(v)
case EncryptedDeviceKey:
storeConfig.EncryptedDevice = (v == "true")
}
}
return storeConfig
}2.2 获取配置的metadataDevice值
func MetadataDevice(config map[string]string) string {
for k, v := range config {
switch k {
case MetadataDeviceKey:
return v
}
}
return ""
}
3. makeJob
3.1 provisionPodTemplateSpec函數
这个比较好理解，就是创建podTemplate结构，主要是volume这块。主要有
- rook-data: dataDirHostPath
- ceph-default-config-dir: emptyDir
- rook-config-override
- rook-ceph-log
- rook-binaries: emptyDir
provisionOSDContainer函数只要是处理devices deviceFilter UseAllDevices这些设备，填充Container结构体
return v1.Container{
Command: []string{path.Join(rookBinariesMountPath, "tini")},
Args: []string{"--", path.Join(rookBinariesMountPath, "rook"), "ceph", "osd", "provision"},
Name: "provision",
Image: c.cephVersion.Image,
VolumeMounts: volumeMounts,
Env: envVars,
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
RunAsUser: &runAsUser,
RunAsNonRoot: &runAsNonRoot,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
},
Resources: resources,
}3.2 runJob
这个就是调用client-go API创建job
func (c *Cluster) runJob(job *batch.Job, nodeName string, config *provisionConfig, action string) bool {
if err := k8sutil.RunReplaceableJob(c.context.Clientset, job); err != nil {
if !errors.IsAlreadyExists(err) {
// we failed to create job, update the orchestration status for this node
message := fmt.Sprintf("failed to create %s job for node %s. %+v", action, nodeName, err)
c.handleOrchestrationFailure(config, nodeName, message)
return false
}
// the job is already in progress so we will let it run to completion
}
logger.Infof("osd %s job started for node %s", action, nodeName)
return true
}
4. completeProvision函数
成功启动osd块服务，这个服务执行
stdbuf -oL ceph-volume lvm activate --no-systemd --filestore 0 6afcbe30-875f-4ff0-b271-ded1392a5610
/bin/mount -t xfs -o rw,noatime,inode64 /dev/ceph-filestore-2214f4f2-0c73-41d5-a467-58287bed012b/osd-data-4c12f679-47e0-4c23-91e5-d98bf320ee42 /var/lib/ceph/osd/ceph-0
/usr/sbin/restorecon /var/lib/ceph/osd/ceph-0
/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
/bin/ln -snf /dev/ceph-filestore-2214f4f2-0c73-41d5-a467-58287bed012b/osd-journal-8fba9aa4-b832-412e-a820-251e5d6b88e4 /var/lib/ceph/osd/ceph-0/journal
/bin/chown -h ceph:ceph /dev/ceph-filestore-2214f4f2-0c73-41d5-a467-58287bed012b/osd-journal-8fba9aa4-b832-412e-a820-251e5d6b88e4
/bin/chown -R ceph:ceph /dev/mapper/ceph--filestore--2214f4f2--0c73--41d5--a467--58287bed012b-osd--journal--8fba9aa4--b832--412e--a820--251e5d6b88e4
ceph-osd --foreground --id 0 --osd-uuid 6afcbe30-875f-4ff0-b271-ded1392a5610 --conf /var/lib/rook/osd0/rook-ceph.config --cluster ceph
4. osd prepare provision
rook ceph osd provision，产生osd配置以及启动osd进程
--cluster-id=885ae2f9-7d26-11e9-82d7-0800271c9f15, --data-device-filter=/dev/sda, --data-devices=, --data-directories=/var/lib/rook, --encrypted-device=false, --force-format=false, --help=false, --location=, --log-flush-frequency=5s, --log-level=INFO, --metadata-device=, --node-name=master-node, --osd-database-size=20480, --osd-journal-size=5120, --osd-store=bluestore, --osd-wal-size=576, --osds-per-device=1
路径： cmd/rook/ceph/osd.go
4.1 根据deviceFilter设置或者不设置，得到devices
var dataDevices []osddaemon.DesiredDevice
if osdDataDeviceFilter != "" {
if cfg.devices != "" {
return fmt.Errorf("Only one of --data-devices and --data-device-filter can be specified.")
}
dataDevices = []osddaemon.DesiredDevice{
{Name: osdDataDeviceFilter, IsFilter: true, OSDsPerDevice: cfg.storeConfig.OSDsPerDevice},
}
} else {
var err error
dataDevices, err = parseDevices(cfg.devices)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to parse device list (%s). %+v", cfg.devices, err))
}
}5. Provision函数
设置初始状态，更新 rook-ceph-osd-%s-status configmap状态为 computingDiff
// set the initial orchestration status
status := oposd.OrchestrationStatus{Status: oposd.OrchestrationStatusComputingDiff}
if err := oposd.UpdateNodeStatus(agent.kv, agent.nodeName, status); err != nil {
return err
}使用 3300 与 6789两个端口
// This tries to detect the current port if the mon already exists
// This basically handles the transtion between monitors running on 6790 to msgr2
// So whatever the previous monitor port was we keep it
currentMonPort := cephutil.GetPortFromEndpoint(monitor.Endpoint)
monPorts := [2]string{strconv.Itoa(int(Msgr2port)), strconv.Itoa(int(currentMonPort))}
msgr1Endpoint := net.JoinHostPort(monIP, monPorts[1])这个就是为什么mon ip加上了v1:
podName := os.Getenv("POD_NAME")
if cluster.CephVersion.IsAtLeastNautilus() {
monHosts[i] = "v1:" + msgr1Endpoint
} else if podName != "" && strings.Contains(podName, "operator") {
// This is an operator and its version is always based on Nautilus
// so it knows how to parse both msgr1 and msgr2 syntax
monHosts[i] = "v1:" + msgr1Endpoint
} else {
// This is not the operator, it's an OSD and its Ceph version is before Nautilus
monHosts[i] = msgr1Endpoint
}实例化CephConfig配置信息，后端存储使用例如rocksdb
conf := &CephConfig{
GlobalConfig: &GlobalConfig{
FSID: cluster.FSID,
RunDir: runDir,
MonMembers: strings.Join(monMembers, " "),
MonHost: strings.Join(monHosts, ","),
PublicAddr: context.NetworkInfo.PublicAddr,
PublicNetwork: context.NetworkInfo.PublicNetwork,
ClusterAddr: context.NetworkInfo.ClusterAddr,
ClusterNetwork: context.NetworkInfo.ClusterNetwork,
MonKeyValueDb: "rocksdb",
MonAllowPoolDelete: true,
MaxPgsPerOsd: 1000,
FileStoreOmapBackend: "rocksdb",
OsdPgBits: 11,
OsdPgpBits: 11,
OsdPoolDefaultSize: 1,
OsdPoolDefaultMinSize: 1,
OsdPoolDefaultPgNum: 100,
OsdPoolDefaultPgpNum: 100,
RbdDefaultFeatures: 3,
FatalSignalHandlers: "false",
},
}5.1 GenerateAdminConnectionConfigWithSettings
产成配置文件client.admin.keyring，允许daemon以admin身份连接
[client.admin]
key = AQDmQeZcq5OoDRAAbDmAoucsY3uk7mTGDAiSOw==
auid = 0
caps mds = "allow *"
caps mon = "allow *"
caps osd = "allow *"
caps mgr = "allow *"
生成配置文件/var/lib/rook/rook-ceph/rook-ceph.config，被拷贝至默认路径/etc/ceph/ceph.conf中
[global]
fsid = dcef92d7-1f6a-4b9d-8ed0-0037d537d00b
run dir = /var/lib/rook/rook-ceph
mon initial members = a
mon host = 10.200.63.69:6789
log file = /dev/stderr
mon cluster log file = /dev/stderr
public addr = 192.170.13.18
cluster addr = 192.170.13.18
mon keyvaluedb = rocksdb
mon_allow_pool_delete = true
mon_max_pg_per_osd = 1000
debug default = 0
debug rados = 0
debug mon = 0
debug osd = 0
debug bluestore = 0
debug filestore = 0
debug journal = 0
debug leveldb = 0
filestore_omap_backend = rocksdb
osd pg bits = 11
osd pgp bits = 11
osd pool default size = 1
osd pool default min size = 1
osd pool default pg num = 100
osd pool default pgp num = 100
crush location = root=default host=node1
rbd_default_features = 3
fatal signal handlers = false[client.admin]
keyring = /var/lib/rook/rook-ceph/client.admin.keyring
DiscoverDevices收集所有设备信息，包括下面几个流程5.2 ListDevices列出所有设备
lsblk --all --noheadings --list --output KNAME
sda
sda1
sda2
sda3
sdb
sr0
func ListDevices(executor exec.Executor) ([]string, error) {
cmd := "lsblk all"
devices, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", "--all", "--noheadings", "--list", "--output", "KNAME")
if err != nil {
return nil, fmt.Errorf("failed to list all devices: %+v", err)
}
return strings.Split(devices, "\n"), nil
}5.3 得到各个设备的属性信息
# lsblk /dev/sda --bytes --nodeps --pairs --output SIZE,ROTA,RO,TYPE,PKNAME
SIZE="42949672960" ROTA="1" RO="0" TYPE="disk" PKNAME=""
func GetDevicePropertiesFromPath(devicePath string, executor exec.Executor) (map[string]string, error) {
cmd := fmt.Sprintf("lsblk %s", devicePath)
output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", devicePath,
"--bytes", "--nodeps", "--pairs", "--output", "SIZE,ROTA,RO,TYPE,PKNAME")
if err != nil {
// try to get more information about the command error
cmdErr, ok := err.(*exec.CommandError)
if ok && cmdErr.ExitStatus() == 32 {
// certain device types (such as loop) return exit status 32 when probed further,
// ignore and continue without logging
return map[string]string{}, nil
}
return nil, err
}
return parseKeyValuePairString(output), nil
}5.4 查询设备的UUID
sgdisk --print /dev/sda
***************************************************************
Found invalid GPT and valid MBR; converting MBR to GPT format
in memory.
***************************************************************Disk /dev/sda: 83886080 sectors, 40.0 GiB
Logical sector size: 512 bytes
Disk identifier (GUID): 015AFD2A-C285-41FE-A729-BDA37B5F30B0
Partition table holds up to 128 entries
First usable sector is 34, last usable sector is 83886046
Partitions will be aligned on 2048-sector boundaries
Total free space is 41945021 sectors (20.0 GiB)Number Start (sector) End (sector) Size Code Name
1 2048 2099199 1024.0 MiB 8300 Linux filesystem
2 2099200 6293503 2.0 GiB 8200 Linux swap
3 6293504 41943039 17.0 GiB 8300 Linux filesystem
// look up the UUID for a disk.
func GetDiskUUID(device string, executor exec.Executor) (string, error) {
if _, err := os.Stat("/usr/sbin/sgdisk"); err != nil {
logger.Warningf("sgdisk not found. skipping disk UUID.")
return "sgdiskNotFound", nil
}
cmd := fmt.Sprintf("get disk %s uuid", device)
output, err := executor.ExecuteCommandWithOutput(false, cmd,
sgdisk, "--print", fmt.Sprintf("/dev/%s", device))
if err != nil {
return "", err
}
return parseUUID(device, output)
}5.5 详细查询设备信息
udevadm info --query=property /dev/sda
DEVLINKS=/dev/disk/by-id/ata-VBOX_HARDDISK_VBbcd43540-cbeefd58 /dev/disk/by-path/pci-0000:00:0d.0-ata-1.0
DEVNAME=/dev/sda
DEVPATH=/devices/pci0000:00/0000:00:0d.0/ata3/host2/target2:0:0/2:0:0:0/block/sda
DEVTYPE=disk
ID_ATA=1
ID_ATA_FEATURE_SET_PM=1
ID_ATA_FEATURE_SET_PM_ENABLED=1
ID_ATA_SATA=1
ID_ATA_SATA_SIGNAL_RATE_GEN2=1
ID_ATA_WRITE_CACHE=1
ID_ATA_WRITE_CACHE_ENABLED=1
ID_BUS=ata
ID_MODEL=VBOX_HARDDISK
ID_MODEL_ENC=VBOX\x20HARDDISK\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20
ID_PART_TABLE_TYPE=dos
ID_PATH=pci-0000:00:0d.0-ata-1.0
ID_PATH_TAG=pci-0000_00_0d_0-ata-1_0
ID_REVISION=1.0
ID_SERIAL=VBOX_HARDDISK_VBbcd43540-cbeefd58
ID_SERIAL_SHORT=VBbcd43540-cbeefd58
ID_TYPE=disk
MAJOR=8
MINOR=0
SUBSYSTEM=block
TAGS=:systemd:
USEC_INITIALIZED=35089
func GetUdevInfo(device string, executor exec.Executor) (map[string]string, error) {
cmd := fmt.Sprintf("udevadm info %s", device)
output, err := executor.ExecuteCommandWithOutput(false, cmd, "udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device))
if err != nil {
return nil, err
}
return parseUdevInfo(output), nil
}5.6 getAvailableDevices
这个要好好分析一下，因为建立很多次，设备没有可用的
5.6.1 设备type如果为part则pass
if device.Type == sys.PartType {
continue
}5.6.2 CheckIfDeviceAvailable
检查设备是否可以用来使用， lsblk /dev/sda --bytes --pairs --output NAME,SIZE,TYPE,PKNAME
分为主设备，分区设备，如果分区ROOK-OSD则为osd可以使用
udevadm info --query=property /dev/sda1 （ID_FS_TYPE=xfs）
GetDeviceFilesystems函数如果含有文件系统则不可以，
6. configureDevice函数
特别乱，乱，能不能整到osdInfo这个结构，留着待分析
6.1 getCephVolumeSupported函数
ceph-volume lvm batch --prepare
6.2 getCephVolumeOSDs
ceph-volume lvm list --format json 列出逻辑设备，查看该节点是否已经配置osd设备
7. startOSDDaemonsOnNode
如果job状态为compled，则启动osd daemon
makeDeployment创建deployment，在创建，具体deployment 如下:
8. completeProvision函数
8.1 completeOSDsForAllNodes
调用 handleStatusConfigMapStatus检查status是否compled
osd prepare job
apiVersion: batch/v1
kind: Job
metadata:
creationTimestamp: "2019-05-23T07:42:48Z"
labels:
app: rook-ceph-osd-prepare
ceph-version: 13.2.3
rook-version: v1.0.1
rook_cluster: rook-ceph
name: rook-ceph-osd-prepare-master-node
namespace: rook-ceph
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
kind: CephCluster
name: rook-ceph
uid: 885ae2f9-7d26-11e9-82d7-0800271c9f15
resourceVersion: "141062"
selfLink: /apis/batch/v1/namespaces/rook-ceph/jobs/rook-ceph-osd-prepare-master-node
uid: 5c3c045b-7d2e-11e9-9875-0800271c9f15
spec:
backoffLimit: 6
completions: 1
parallelism: 1
selector:
matchLabels:
controller-uid: 5c3c045b-7d2e-11e9-9875-0800271c9f15
template:
metadata:
creationTimestamp: null
labels:
app: rook-ceph-osd-prepare
controller-uid: 5c3c045b-7d2e-11e9-9875-0800271c9f15
job-name: rook-ceph-osd-prepare-master-node
rook_cluster: rook-ceph
name: rook-ceph-osd
spec:
affinity: {}
containers:
- args:
- ceph
- osd
- copybins
env:
- name: ROOK_PATH
value: /rook
image: rook/ceph:v1.0.1
imagePullPolicy: IfNotPresent
name: copy-bins
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /rook
name: rook-binaries
- args:
- --
- /rook/rook
- ceph
- osd
- provision
command:
- /rook/tini
env:
- name: ROOK_NODE_NAME
value: master-node
- name: ROOK_CLUSTER_ID
value: 885ae2f9-7d26-11e9-82d7-0800271c9f15
- name: ROOK_PRIVATE_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: ROOK_PUBLIC_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: ROOK_CLUSTER_NAME
value: rook-ceph
- name: ROOK_MON_ENDPOINTS
valueFrom:
configMapKeyRef:
key: data
name: rook-ceph-mon-endpoints
- name: ROOK_MON_SECRET
valueFrom:
secretKeyRef:
key: mon-secret
name: rook-ceph-mon
- name: ROOK_ADMIN_SECRET
valueFrom:
secretKeyRef:
key: admin-secret
name: rook-ceph-mon
- name: ROOK_CONFIG_DIR
value: /var/lib/rook
- name: ROOK_CEPH_CONFIG_OVERRIDE
value: /etc/rook/config/override.conf
- name: ROOK_FSID
valueFrom:
secretKeyRef:
key: fsid
name: rook-ceph-mon
- name: ROOK_OSD_STORE
value: bluestore
- name: ROOK_OSDS_PER_DEVICE
value: "1"
- name: ROOK_DATA_DEVICE_FILTER
value: /dev/sda
- name: ROOK_DATA_DIRECTORIES
value: /var/lib/rook
image: ceph/ceph:v13.2.3-20190410
imagePullPolicy: IfNotPresent
name: provision
resources: {}
securityContext:
privileged: true
procMount: Default
readOnlyRootFilesystem: false
runAsNonRoot: false
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/lib/rook
name: rook-data
- mountPath: /etc/ceph
name: ceph-default-config-dir
- mountPath: /var/log/ceph
name: rook-ceph-log
- mountPath: /rook
name: rook-binaries
- mountPath: /dev
name: devices
- mountPath: /run/udev
name: udev
dnsPolicy: ClusterFirst
nodeSelector:
kubernetes.io/hostname: master-node
restartPolicy: OnFailure
schedulerName: default-scheduler
securityContext: {}
serviceAccount: rook-ceph-osd
serviceAccountName: rook-ceph-osd
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /var/lib/rook
type: ""
name: rook-data
- emptyDir: {}
name: ceph-default-config-dir
- configMap:
defaultMode: 420
items:
- key: config
path: override.conf
name: rook-config-override
name: rook-config-override
- hostPath:
path: /var/lib/rook/log/rook-ceph
type: ""
name: rook-ceph-log
- emptyDir: {}
name: rook-binaries
- hostPath:
path: /dev
type: ""
name: devices
- hostPath:
path: /run/udev
type: ""
name: udev
status:
completionTime: "2019-05-23T07:43:07Z"
conditions:
- lastProbeTime: "2019-05-23T07:43:07Z"
lastTransitionTime: "2019-05-23T07:43:07Z"
status: "True"
type: Complete
startTime: "2019-05-23T07:42:48Z"
succeeded: 1
osd deployment
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
creationTimestamp: "2019-05-23T06:47:35Z"
generation: 7
labels:
app: rook-ceph-osd
ceph-osd-id: "0"
ceph-version: 13.2.3
rook-version: v1.0.1
rook_cluster: rook-ceph
name: rook-ceph-osd-0
namespace: rook-ceph
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
kind: CephCluster
name: rook-ceph
uid: 885ae2f9-7d26-11e9-82d7-0800271c9f15
resourceVersion: "363313"
selfLink: /apis/extensions/v1beta1/namespaces/rook-ceph/deployments/rook-ceph-osd-0
uid: a5fd34bd-7d26-11e9-82d7-0800271c9f15
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: rook-ceph-osd
ceph-osd-id: "0"
rook_cluster: rook-ceph
strategy:
type: Recreate
template:
metadata:
creationTimestamp: null
labels:
app: rook-ceph-osd
ceph-osd-id: "0"
rook_cluster: rook-ceph
name: rook-ceph-osd
spec:
affinity: {}
containers:
- args:
- --foreground
- --id
- "0"
- --conf
- /var/lib/rook/osd0/rook-ceph.config
- --osd-data
- /var/lib/rook/osd0
- --keyring
- /var/lib/rook/osd0/keyring
- --cluster
- rook-ceph
- --osd-uuid
- 210b60a6-6661-4d46-8cf7-aa3591540d8b
command:
- ceph-osd
env:
- name: ROOK_NODE_NAME
value: master-node
- name: ROOK_PRIVATE_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: ROOK_PUBLIC_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: TINI_SUBREAPER
- name: CONTAINER_IMAGE
value: ceph/ceph:v13.2.3-20190410
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: POD_MEMORY_LIMIT
valueFrom:
resourceFieldRef:
divisor: "0"
resource: limits.memory
- name: POD_MEMORY_REQUEST
valueFrom:
resourceFieldRef:
divisor: "0"
resource: requests.memory
- name: POD_CPU_LIMIT
valueFrom:
resourceFieldRef:
divisor: "1"
resource: limits.cpu
- name: POD_CPU_REQUEST
valueFrom:
resourceFieldRef:
divisor: "0"
resource: requests.cpu
- name: ROOK_OSD_UUID
value: 210b60a6-6661-4d46-8cf7-aa3591540d8b
- name: ROOK_OSD_ID
value: "0"
- name: ROOK_OSD_STORE_TYPE
value: bluestore
image: ceph/ceph:v13.2.3-20190410
imagePullPolicy: IfNotPresent
name: osd
resources: {}
securityContext:
privileged: true
procMount: Default
readOnlyRootFilesystem: false
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/lib/rook
name: rook-data
- mountPath: /etc/ceph
name: ceph-default-config-dir
- mountPath: /var/log/ceph
name: rook-ceph-log
- mountPath: /rook
name: rook-binaries
dnsPolicy: ClusterFirst
hostPID: true
initContainers:
- args:
- ceph
- osd
- init
env:
- name: ROOK_NODE_NAME
value: master-node
- name: ROOK_CLUSTER_ID
value: 885ae2f9-7d26-11e9-82d7-0800271c9f15
- name: ROOK_PRIVATE_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: ROOK_PUBLIC_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: ROOK_CLUSTER_NAME
value: rook-ceph
- name: ROOK_MON_ENDPOINTS
valueFrom:
configMapKeyRef:
key: data
name: rook-ceph-mon-endpoints
- name: ROOK_MON_SECRET
valueFrom:
secretKeyRef:
key: mon-secret
name: rook-ceph-mon
- name: ROOK_ADMIN_SECRET
valueFrom:
secretKeyRef:
key: admin-secret
name: rook-ceph-mon
- name: ROOK_CONFIG_DIR
value: /var/lib/rook
- name: ROOK_CEPH_CONFIG_OVERRIDE
value: /etc/rook/config/override.conf
- name: ROOK_FSID
valueFrom:
secretKeyRef:
key: fsid
name: rook-ceph-mon
- name: ROOK_OSD_STORE
value: bluestore
- name: ROOK_OSDS_PER_DEVICE
value: "1"
- name: TINI_SUBREAPER
- name: ROOK_OSD_ID
value: "0"
- name: ROOK_CEPH_VERSION
value: ceph version 13.2.3 mimic
image: rook/ceph:v1.0.1
imagePullPolicy: IfNotPresent
name: config-init
resources: {}
securityContext:
privileged: true
procMount: Default
readOnlyRootFilesystem: false
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/lib/rook
name: rook-data
- mountPath: /etc/ceph
name: ceph-default-config-dir
- mountPath: /var/log/ceph
name: rook-ceph-log
- mountPath: /etc/rook/config
name: rook-config-override
- args:
- ceph
- osd
- copybins
env:
- name: ROOK_PATH
value: /rook
image: rook/ceph:v1.0.1
imagePullPolicy: IfNotPresent
name: copy-bins
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /rook
name: rook-binaries
nodeSelector:
kubernetes.io/hostname: master-node
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: rook-ceph-osd
serviceAccountName: rook-ceph-osd
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /var/lib/rook
type: ""
name: rook-data
- emptyDir: {}
name: ceph-default-config-dir
- configMap:
defaultMode: 420
items:
- key: config
path: override.conf
name: rook-config-override
name: rook-config-override
- hostPath:
path: /var/lib/rook/log/rook-ceph
type: ""
name: rook-ceph-log
- emptyDir: {}
name: rook-binaries
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2019-05-23T06:47:35Z"
lastUpdateTime: "2019-05-23T06:47:44Z"
message: ReplicaSet "rook-ceph-osd-0-7bd86957b9" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
- lastTransitionTime: "2019-05-23T07:41:51Z"
lastUpdateTime: "2019-05-23T07:41:51Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
observedGeneration: 7
readyReplicas: 1
replicas: 1
updatedReplicas: 1
总结：
osd启动流程：
创建osd prepare job，这个是用来产生osd配置， 各种查设备信息，命令特别多
如果job状态为compled，则创建osd deployment