centos7.1中安装calamari
ceph集群安装
-
确定版本
[root@node1050 calamari]# ceph --version ceph version 10.2.9 (2ee413f77150c0f375ff6f10edd6c8f9c7d060d0) [root@node1050 calamari]# -
配置无密访问
-
准备脚本
[root@node1050 17-ceph-install]# cat reinstall.sh ceph-deploy purge node1017 node1018 node1019 ceph-deploy purgedata node1017 node1018 node1019 ceph-deploy forgetkeys rm ceph.* cp -a /root/yum.repos_bak/* /etc/yum.repos.d/ scp -r /etc/yum.repos.d/* 192.168.80.18:/etc/yum.repos.d/ scp -r /etc/yum.repos.d/* 192.168.80.19:/etc/yum.repos.d/ ssh 192.168.80.17 "yum -y install epel-release ceph ceph-radosgw" & ssh 192.168.80.18 "yum -y install epel-release ceph ceph-radosgw" & ssh 192.168.80.19 "yum -y install epel-release ceph ceph-radosgw" & wait ceph-deploy new node1017 echo public network = 10.10.10.0/24 >> ceph.conf echo osd pool default size = 2 >> ceph.conf echo osd pool default size = 2 >> ceph.conf echo osd pool default pg num = 64 >> ceph.conf echo osd pool default pgp num = 64 >> ceph.conf wait ceph-deploy install node1017 node1018 node1019 ceph-deploy mon create-initial ceph-deploy admin node1017 node1018 node1019 sh zap.sh sh zap2.sh sh prepare.sh sh prepare2.sh sh active.sh sh active2.sh[root@node1050 17-ceph-install]# cat active.sh i=1 #for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi" #do # ceph-deploy osd activate node1017:/dev/$dev"1":/dev/sdb$i # ceph-deploy osd activate node1018:/dev/$dev"1":/dev/sdb$i # ceph-deploy osd activate node1019:/dev/$dev"1":/dev/sdb$i # i=$(($i+1)) #done ceph-deploy osd activate node1017:/dev/sdb1 ceph-deploy osd activate node1018:/dev/sdc1 ceph-deploy osd activate node1019:/dev/sdc1[root@node1050 17-ceph-install]# cat active2.sh i=1 for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi" "sdj" "sdk" "sdl" "sdm" "sdn" "sdo" "sdp" "sdq" "sdr" "sds" "sdt" "sdu" "sdv" "sdw" "sdx" "sdy" do ceph-deploy osd activate node1017:"/dev/$dev"1 ceph-deploy osd activate node1018:"/dev/$dev"1 ceph-deploy osd activate node1019:"/dev/$dev"1 done[root@node1050 17-ceph-install]# cat create_fs.sh ceph osd pool create cephfs_data 128 ceph osd pool create cephfs_metadata 64 ceph fs new ceph_fs cephfs_metadata cephfs_data cat cat ceph.client.admin.keyring > /etc/admin.sec[root@node1050 17-ceph-install]# cat zap.sh for dev in "sdb" "sdc" do echo -------------------------$dev ceph-deploy disk zap node1017:$dev ceph-deploy disk zap node1018:$dev ceph-deploy disk zap node1019:$dev done[root@node1050 17-ceph-install]# cat zap2.sh for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi" "sdj" "sdk" "sdl" "sdm" "sdn" "sdo" "sdp" "sdq" "sdr" "sds" "sdt" "sdu" "sdv" "sdw" "sdx" "sdy" do echo -------------------------$dev ceph-deploy disk zap node1017:$dev ceph-deploy disk zap node1018:$dev ceph-deploy disk zap node1019:$dev done[root@node1050 17-ceph-install]# cat prepare.sh ceph-deploy osd prepare node1017:sdb ceph-deploy osd prepare node1018:sdc ceph-deploy osd prepare node1019:sdc[root@node1050 17-ceph-install]# cat prepare2.sh for dev in "sdd" "sde" "sdf" "sdg" "sdh" "sdi" "sdj" "sdk" "sdl" "sdm" "sdn" "sdo" "sdp" "sdq" "sdr" "sds" "sdt" "sdu" "sdv" "sdw" "sdx" "sdy" do ceph-deploy osd prepare node1017:$dev ceph-deploy osd prepare node1018:$dev ceph-deploy osd prepare node1019:$dev done
calamari服务端安装
git clone https://github.com/ceph/calamari.git
yum install gcc gcc-c++ postgresql-libs python-virtualenv postgresql-devel httpd checkpolicy selinux-policy-devel selinux-policy-doc selinux-policy-mls rpm-build redhat-lsb-core cairo-devel.x86_64
cd calamari/
git checkout v1.3.1.1
./build-rpm.sh
cd ../rpmbuild/RPMS/x86_64/
yum install calamari-server-1.5.2-13_g768f37d.el7.centos.x86_64.rpm
calamari-ctl initialize
calamari-ctl add_user --password 111111 --email [email protected] root
calamari-ctl assign_role --role superuser root
安装过程中遇到的问题
编译过程提示/etc/calamari/calamari.conf is a directory
修改如下内容
[root@node1019 calamari]# git diff
diff --git a/Makefile b/Makefile
index 5e51859..ae1526a 100644
--- a/Makefile
+++ b/Makefile
@@ -166,9 +166,8 @@ install-conf: $(CONFFILES)
@$(INSTALL) -d $(DESTDIR)/var/lib/graphite/whisper
@$(INSTALL) -d $(DESTDIR)/var/lib/calamari
@$(INSTALL) -d $(DESTDIR)/var/lib/cthulhu
-
@$(INSTALL) -d $(DESTDIR)/etc/calamari
- @$(INSTALL) -D -m 0644 conf/calamari/$(FLAVOR)/calamari.conf \
+ @$(INSTALL) -D -m 0644 conf/calamari/el6/calamari.conf \
$(DESTDIR)/etc/calamari/calamari.conf
@$(INSTALL) -D -m 0644 conf/alembic.ini \
$(DESTDIR)/etc/calamari/alembic.ini
编译过程中提示语法错误
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-R57qBkSe-1592052236921)(/images/calamari_build_err.png)]
这是因为使用的gevent1.2.2版本中的_socket3.py文件中使用了python3的语法
diff --git a/requirements/2.7/requirements.production.txt b/requirements/2.7/requirements.production.txt
index 41c214c..8e4acb4 100644
--- a/requirements/2.7/requirements.production.txt
+++ b/requirements/2.7/requirements.production.txt
@@ -12,7 +12,7 @@ wsgiref==0.1.2
sqlalchemy==0.8.3
alembic==0.6.3
cython==0.19.2
-gevent>=1.0
+gevent==1.0
greenlet>=0.3.2
psycogreen==1.0
psycopg2==2.5.2
diff --git a/requirements/2.7/requirements.txt b/requirements/2.7/requirements.txt
index 146b772..f02476d 100644
--- a/requirements/2.7/requirements.txt
+++ b/requirements/2.7/requirements.txt
@@ -25,7 +25,7 @@ psutil==2.0.0
# Gevent 1.0.x requires cython but doesn't do a good job of depending on it
cython==0.19.2
-gevent>=1.0
+gevent==1.0
greenlet>=0.3.2
/var/log/calamari/calamari.log中有错误信息
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-4dRoAPyj-1592052236923)(/images/calamari_log_err.png)]
需要安装cairo包yum install cairo-devel.x86_64
calamari客户端安装
1 下载源码
git clone https://github.com/ceph/romana
2 安装依赖包
yum install npm ruby ruby-devel rubygems libpng-devel libpng-devel pngquant optipng libjpeg
3 修改npm仓库地址并下载依赖包
npm config set registry https://registry.npm.taobao.org
npm config get registry
npm install -g bower grunt grunt-cli grunt-contrib-compass
npm install -g n
npm install --global jpegtran-bin
npm install --global gifsicle
gem sources --remove https://rubygems.org/
gem sources --remove gems.ruby-china.org
gem sources -a https://ruby.taobao.org
gem update --system
gem install compass
cd romana/
git checkout v1.2.2
[root@calamari ~]# vim romana/manage/package.json
"grunt-connect-proxy": "~0.2.0"
cp /usr/local/bin/compass /sbin/
4 修改grunt-contrib-imagemin版本号
[root@node1050 romana]# grep "grunt-contrib-imagemin" -R *
admin/package.json: "grunt-contrib-imagemin": "~0.1.4",
dashboard/package.json: "grunt-contrib-imagemin": "0.1.4",
login/package.json: "grunt-contrib-imagemin": "~0.1.3",
manage/package.json: "grunt-contrib-imagemin": "~0.3.0",
[root@node1050 romana]# vim admin/package.json
[root@node1050 romana]# vim dashboard/package.json
[root@node1050 romana]# vim login/package.json
[root@node1050 romana]# vim manage/package.json
[root@node1050 romana]# grep "grunt-contrib-imagemin" -R *
admin/package.json: "grunt-contrib-imagemin": "0.4.1",
dashboard/package.json: "grunt-contrib-imagemin": "0.4.1",
login/package.json: "grunt-contrib-imagemin": "0.4.1",
manage/package.json: "grunt-contrib-imagemin": "0.4.1",
5 修改yo的版本号为2.0
[root@node1050 romana]# grep "\"yo\":" * -R
admin/package.json: "yo": "2.0.0",
admin/node_modules/yo/package.json: "yo": "lib/cli.js",
admin/node_modules/yo/package.json: "yo": [
admin/node_modules/generator-angular/package.json: "yo": ">=1.0.0-rc.1.1"
admin/node_modules/tabtab/examples/yo-complete/package.json: "yo": [
dashboard/node_modules/generator-backbone/package.json: "yo": ">=1.0.0-rc.1.1"
login/package.json: "yo": "2.0.0",
login/node_modules/yo/package.json: "yo": "lib/cli.js",
login/node_modules/yo/package.json: "yo": [
login/node_modules/tabtab/examples/yo-complete/package.json: "yo": [
login/node_modules/generator-backbone/package.json: "yo": ">=1.0.0-rc.1.1"
6 执行编译
PHANTOMJS_CDNURL=http://npm.taobao.org/dist/phantomjs make build-real
7 编译完成后打包
[root@calamari romana]# make dist
making dist tarball in ../romana_1.2.2.tar.gz for d in manage admin login dashboard; do \ echo $d; \ (cd $d; \ npm install --silent; \ grunt --no-color saveRevision) \ done --------something more---------- Done, without errors. 185667 blocks tar file made in ../romana_1.2.2.tar.gz
cp ../calamari-clients_1.2.2.tar.gz ./
8 修改makefile
[root@node1050 romana]# git diff Makefile
diff --git a/Makefile b/Makefile
index 420a94a..add506e 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ SRC := $(shell pwd)
# set these only if not set with ?=
VERSION ?= $(shell $(SRC)/get-versions.sh VERSION)
REVISION ?= $(shell $(SRC)/get-versions.sh REVISION)
-BUILD_PRODUCT_TGZ=$(SRC)/calamari-clients-build-output.tar.gz
+BUILD_PRODUCT_TGZ=$(SRC)/calamari-clients_1.2.2.tar.gz
RPM_REVISION ?= $(shell $(SRC)/get-versions.sh -r REVISION)
RPMBUILD=$(SRC)/../rpmbuild
9 修改client.spec
[root@node1050 romana]# git diff clients.spec
diff --git a/clients.spec b/clients.spec
index 6cea00b..b53fd79 100644
--- a/clients.spec
+++ b/clients.spec
@@ -31,6 +31,15 @@ echo "install"
mkdir -p %{buildroot}
cd %{buildroot}
tar xfz %{tarname}
+cd %{name}-%{version}
+for dir in manage admin login dashboard
+do
+mkdir -p ../opt/calamari/webapp/content/"$dir"
+cp -pr "$dir"/dist/* ../opt/calamari/webapp/content/"$dir"/
+done
+cd ../
+rm -rf /tmp/%{name}-%{version}
+mv %{name}-%{version} /tmp/
%clean
echo "clean"
10 安装rpm
cd /root/rpmbuild/RPMS/x86_64/
yum install calamari-clients-1.2.2-0.el7.centos.x86_64.rpm
安装过程中遇到的问题
yum npm安装之后再执行npm install -g npm会把npm删掉
npm WARN [email protected] requires a peer of grunt@~0.4.0 but none was installed
在执行gem update --system之前不光要删除gem sources --remove https://rubygems.org还要删除https://gems.ruby-china.org
calamari-client 执行make build-real过程中会提示没有pngquant，需要手动安装
yum install npm ruby ruby-devel rubygems libpng-devel libpng-devel pngquant optipng libjpeg
yum install optipng
yum install libjpeg
npm install --global jpegtran-bin
npm install --global gifsicle
在执行npm时有错误提示：
npm: relocation error: npm: symbol SSL_set_cert_cb, version libssl.so.10 not defined in file libssl.so.10 with link time reference
此时需要升级openssl，执行：
yum update openssl
需要修改下面的文件中
login/node_modules/phantomjs/install.js
manage/node_modules/phantomjs/install.js
dashboard/node_modules/phantomjs/install.js
替换phantomjs-1.9.8-linux-x86_64.tar.bz2下载路径
//var DEFAULT_CDN = 'https://github.com/Medium/phantomjs/releases/download/v1.9.19'
var DEFAULT_CDN = 'http://npm.taobao.org/mirrors/phantomjs'
diamond安装
1 下载源码
git clone https://github.com/ceph/Diamond
2 编译安装
cd Diamond
git checkout origin/calamari
make rpm
yum install ./dist/diamond-3.4.67-0.noarch.rpm
3 修改配置diamond.conf
cp /etc/diamond/diamond.conf.example /etc/diamond/diamond.conf
[root@node1050 Diamond]# diff /etc/diamond/diamond.conf.example /etc/diamond/diamond.conf
59c59
< host = graphite
---
> host = node1050
74c74
< host = graphite
---
> host = node1050
4 启动diamond
/etc/init.d/diamond restart
5 修改salt-minon配置并启动
[root@node1050 Diamond]# mkdir /etc/salt/minion.d
[root@node1050 Diamond]# echo "master: node1050" > /etc/salt/minion.d/calamari.conf
[root@node1050 Diamond]# systemctl restart salt-minion
7 其他节点啊装minion
将diamond-3.4.67-0.noarch.rpm拷贝到其他ceph节点并执行安装，修改配置及启动
在其他ceph节点安装salt-minion并修改配置启动
8 系统启动
在calamari-server节点执行
salt-key -L
salt-key -A
chmod 777 -R /var/log/calamari/*
systemctl restart supervisord
systemctl restart httpd
重新安装slat
http://rpm.pbone.net/ 中查找（rpmfind.net中已经没有该版本的包）
salt-2014.1.13-1.fc21.noarch.rpm
salt-master-2014.1.13-1.fc21.noarch.rpm
salt-minion-2014.1.13-1.fc21.noarch.rpm
遇到的问题
界面未显示登录界面，显示内部错误
查看/var/log/calamari/calamari.log无内容显示，查看/var/log/calamari/cthulhu.log有如下错误
conn = bind.contextual_connect()
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1661, in contextual_connect
self.pool.connect(),
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 272, in connect
return _ConnectionFairy(self).checkout()
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 431, in __init__
rec = self._connection_record = pool._do_get()
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 788, in _do_get
con = self._create_connection()
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 225, in _create_connection
return _ConnectionRecord(self)
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 318, in __init__
self.connection = self.__connect()
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/pool.py", line 379, in __connect
connection = self.__pool._creator()
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/engine/strategies.py", line 80, in connect
return dialect.connect(*cargs, **cparams)
File "/opt/calamari/venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 283, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/opt/calamari/venv/lib/python2.7/site-packages/psycopg2/__init__.py", line 164, in connect
conn = _connect(dsn, connection_factory=connection_factory, async=async)
File "/opt/calamari/venv/lib/python2.7/site-packages/psycogreen/gevent.py", line 29, in gevent_wait_callback
state = conn.poll()
OperationalError: (OperationalError) could not connect to server: Connection refused
Is the server running on host "localhost" (::1) and accepting
TCP/IP connections on port 5432?
could not connect to server: Connection refused
Is the server running on host "localhost" (127.0.0.1) and accepting
TCP/IP connections on port 5432?
None None
2017-10-23 10:49:32,094 - ERROR - cthulhu Recovery failed
执行setenforce 0关闭selinux
界面一直显示为初次安装
发现调用cluster接口的时候返回的cluster信息为空，所以前端认为没有集群信息
执行salt ‘*’ ceph.get_heartbeats出现下面的错误
修改代码/opt/calamari/salt/salt/_modules/ceph.py
修改上面的代码之后，再次调用接口可以获取正确的cluster信息
界面无法正常显示
/var/log/calamari/calamari.log中出现如下错误
Traceback (most recent call last):
File "/opt/calamari/venv/lib/python2.7/site-packages/django/core/handlers/base.py", line 115, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "/opt/calamari/venv/lib/python2.7/site-packages/rest_framework/viewsets.py", line 78, in view
return self.dispatch(request, *args, **kwargs)
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/rpc_view.py", line 92, in dispatch
return super(RPCViewSet, self).dispatch(request, *args, **kwargs)
File "/opt/calamari/venv/lib/python2.7/site-packages/django/views/decorators/csrf.py", line 77, in wrapped_view
return view_func(*args, **kwargs)
File "/opt/calamari/venv/lib/python2.7/site-packages/rest_framework/views.py", line 399, in dispatch
response = self.handle_exception(exc)
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/rpc_view.py", line 109, in handle_exception
return super(RPCViewSet, self).handle_exception(exc)
File "/opt/calamari/venv/lib/python2.7/site-packages/rest_framework/views.py", line 396, in dispatch
response = handler(request, *args, **kwargs)
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py", line 316, in get
osd_data = osd_data.get()
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py", line 167, in generate
'mds': cls._calculate_mds_counters(mds_map),
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py", line 295, in _calculate_mds_counters
#up = len(mds_map['up'])
TypeError: 'NoneType' object has no attribute '__getitem__'
修改代码/opt/calamari/venv/lib/python2.7/site-packages/calamari_rest_api-0.1-py2.7.egg/calamari_rest/views/v1.py
[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-o3J8YKEG-1592052236927)(/images/v1.png)]
/var/log/calamari/cthulhu.log中有如下错误
2017-10-23 19:37:00,644 - ERROR - cthulhu Exception handling message with tag ceph/cluster/354c211d-e8d1-4dcd-8635-f0fb7ec51826
Traceback (most recent call last):
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/manager/cluster_monitor.py", line 244, in _run
self.on_heartbeat(data['id'], data['data'])
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/gevent_util.py", line 35, in wrapped
return func(*args, **kwargs)
File "/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/manager/cluster_monitor.py", line 348, in on_heartbeat
cluster_data['versions'][sync_type.str])
KeyError: 'mds_map'
修改代码
/opt/calamari/venv/lib/python2.7/site-packages/calamari_cthulhu-0.1-py2.7.egg/cthulhu/manager/cluster_monitor.py
/var/log/diamond/diamond.log中有错误
出现断言错误
修改代码/usr/share/diamond/collectors/ceph/ceph.py
参考资料
- 《ceph分布式存储实战》
- https://zphj1987.gitbooks.io/calamaribook/content/second/an_zhuang_calamari_-_server.html
- http://blog.csdn.net/lzw06061139/article/details/51174588
声明：转载本博文章须在文章明显处注明作者及附上原文链接，便于读者找到原文的更新版。