heketi简介
heketi：提供基于RESTful接口管理glusterfs的功能，可以方便的创建集群管理glusterfs的node，device，volume；与k8s结合可以创建动态的PV，扩展glusterfs存储的动态管理功能
heketi-cli：heketi的客户端工具
本文基于手工部署的方式熟悉heketi的架构和使用
环境
| 服务器 | IP | 角色 |
|---|---|---|
| master-192 | 172.30.81.192 | gluster-node,heketi server |
| node-193 | 172.30.81.193 | gluster-node |
| node-194 | 172.30.81.194 | heketi-cli |
注：master-192,node-193已经部署好glusterfs集群
安装heketi
master-192
yum install -y heketi
创建ssh用户免密钥登录glusterfs节点（以root为例）
ssh-keygen -f /etc/heketi/heketi_key -t rsa -N ''
[root@master-192 heketi]# ll /etc/heketi/
总用量 204
-rw-r--r-- 1 root root 1789 11月 9 15:13 heketi.json
-rw------- 1 root root 1679 11月 9 14:58 heketi_key
-rw-r--r-- 1 root root 397 11月 9 14:58 heketi_key.pub
将公钥放到对应节点
ssh-copy-id -i /etc/heketi/heketi_key.pub [email protected]
ssh-copy-id -i /etc/heketi/heketi_key.pub [email protected]
验证登录无需输入密码
[root@master-192 heketi]# ssh -i /etc/heketi/heketi_key [email protected]
Last login: Fri Nov 9 16:13:36 2018 from master-192
[root@node-193 ~]#
修改heketi的配置文件/etc/heketi/heketi.json
{
"_port_comment": "Heketi Server Port Number",
"port": "8080",
"_use_auth": "Enable JWT authorization. Please enable for deployment",
"use_auth": false,
"_jwt": "Private keys for access",
"jwt": {
"_admin": "Admin has access to all APIs",
"admin": {
"key": "123456"
},
"_user": "User only has access to /volumes endpoint",
"user": {
"key": "123456"
}
},
"_glusterfs_comment": "GlusterFS Configuration",
"glusterfs": {
"_executor_comment": [
"Execute plugin. Possible choices: mock, ssh",
"mock: This setting is used for testing and development.",
" It will not send commands to any node.",
"ssh: This setting will notify Heketi to ssh to the nodes.",
" It will need the values in sshexec to be configured.",
"kubernetes: Communicate with GlusterFS containers over",
" Kubernetes exec api."
],
"executor": "ssh",
"_sshexec_comment": "SSH username and private key file information",
"sshexec": {
"keyfile": "/etc/heketi/heketi_key",
"user": "root"
},
"_kubeexec_comment": "Kubernetes configuration",
"kubeexec": {
"host" :"https://kubernetes.host:8443",
"cert" : "/path/to/crt.file",
"insecure": false,
"user": "kubernetes username",
"password": "password for kubernetes user",
"namespace": "OpenShift project or Kubernetes namespace",
"fstab": "Optional: Specify fstab file on node. Default is /etc/fstab"
},
"_db_comment": "Database file name",
"db": "/var/lib/heketi/heketi.db",
"_loglevel_comment": [
"Set log level. Choices are:",
" none, critical, error, warning, info, debug",
"Default is warning"
],
"loglevel" : "debug"
}
}
注："executor": ssh生产环境使用，kubernetes 为glusterfs容器部署使用
use_auth: true 开启认证模式，调用添加--user=admin --secret=123456
修改/usr/lib/systemd/system/heketi.service，否则通过service启动报错
[Unit]
Description=Heketi Server
[Service]
Type=simple
WorkingDirectory=/var/lib/heketi
EnvironmentFile=-/etc/heketi/heketi.json
User=root
ExecStart=/usr/bin/heketi --config=/etc/heketi/heketi.json
Restart=on-failure
StandardOutput=syslog
StandardError=syslog
[Install]
WantedBy=multi-user.target
启动heketi
systemctl daemon-reload
systemctl start heketi
验证
[root@master-192 diff]# curl http://172.30.81.192:8080/hello
Hello from Heketi[root@master-192 diff]#
hekeli-cli部署使用
node-194
yum install -y heketi-cli
glutserfs集群数据主要包含节点node，brick存储点device，下面创建heketi集群的topology
[root@node-194 ~]# cat /etc/heketi/heketi-topology.json
{
"clusters": [
{
"nodes": [
{
"node": {
"hostnames": {
"manage": [
"172.30.81.192"
],
"storage": [
"172.30.81.192"
]
},
"zone": 1
},
"devices": [
"/dev/vdb"
]
},
{
"node": {
"hostnames": {
"manage": [
"172.30.81.193"
],
"storage": [
"172.30.81.193"
]
},
"zone": 1
},
"devices": [
"/dev/vdb"
]
}
]
}
]
}
[root@node-194 ~]# heketi-cli --server http://172.30.81.192:8080 topology load --json=/etc/heketi/heketi-topology.json
Creating cluster ... ID: a9ca2cbc28c1194c59c5e26aac3ee307
Allowing file volumes on cluster.
Allowing block volumes on cluster.
Creating node 172.30.81.192 ... ID: 8ef15510fb3152ab4515a375474842e3
Adding device /dev/vdb ... OK
Creating node 172.30.81.193 ... ID: 013d0fbed34f01964243f91123347568
Adding device /dev/vdb ... OK
集群创建完成查看数据
[root@node-194 ~]# heketi-cli --server http://172.30.81.192:8080 cluster list
Clusters:
Id:a9ca2cbc28c1194c59c5e26aac3ee307 [file][block]
[root@node-194 ~]# heketi-cli --server http://172.30.81.192:8080 node list
Id:013d0fbed34f01964243f91123347568 Cluster:a9ca2cbc28c1194c59c5e26aac3ee307
Id:8ef15510fb3152ab4515a375474842e3 Cluster:a9ca2cbc28c1194c59c5e26aac3ee307
下面通过heketi创建gluster volume
[root@node-194 ~]# heketi-cli --server http://172.30.81.192:8080 volume create --size=10 --replica=2
Name: vol_244ebb5ee623b28a18ace5c39db721ab
Size: 10
Volume Id: 244ebb5ee623b28a18ace5c39db721ab
Cluster Id: a9ca2cbc28c1194c59c5e26aac3ee307
Mount: 172.30.81.193:vol_244ebb5ee623b28a18ace5c39db721ab
Mount Options: backup-volfile-servers=172.30.81.192
Block: false
Free Size: 0
Block Volumes: []
Durability Type: replicate
Distributed+Replica: 2
glutserfs集群上查看volume
[root@master-192 diff]# gluster volume info
Volume Name: vol_244ebb5ee623b28a18ace5c39db721ab
Type: Replicate
Volume ID: e325399b-b458-4f88-b4d9-420c0082cf78
Status: Started
Snapshot Count: 0
Number of Bricks: 1 x 2 = 2
Transport-type: tcp
Bricks:
Brick1: 172.30.81.192:/var/lib/heketi/mounts/vg_10ff1dfd97b93c2f4a19bc51628d9581/brick_cf81dcf6916ec28c2ba8d837621c4a53/brick
Brick2: 172.30.81.193:/var/lib/heketi/mounts/vg_d78562d163b20e0b20083b5776f47df3/brick_bf99a18af00887c0e9879481848d5712/brick
Options Reconfigured:
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off