Centos7.5安装配置Greenplum5.7

一、所有主机都需要做的配置操作

在开始之前，要做以下准备工作：
- 将用户切换为root用户，并且知道root账户的登陆密码（例如：123qwe!@#）。
- 使用系统自带的python2.7版本，如果安装有anaconda等其他版本的python，请先从环境变量中移除。

1.关闭防火墙，确保各个主机可以访问端口号（所有主机）

# iptables -F
# setenforce 0
# sed -ri  "/^SELINUX/cSELINUX=disabled" /etc/selinux/config
# systemctl stop firewalld
# systemctl disable firewalld

2.修改域名解析/etc/hosts（所有主机）

master,segment参考https://greenplum.org/docs/580/admin_guide/intro/arch_overview.html

# vi /etc/hosts #在文件底部追加
# 参考文章https://www.linuxidc.com/Linux/2016-10/135886.htm
# 格式为：
# ip地址 主机名（可在shell中通过hostname命令查看） 别名（自定义的名字）
# 注意：
# 各主机名需修改至唯一，不可采用默认主机名
# ip hostname alias
192.168.237.13 hostname13 node1-master
192.168.237.14 hostname14 node2-seg node2-standby
192.168.237.15 hostname15 node3-seg

3.修改文件打开限制（所有主机）

# vi /etc/security/limits.conf #追加
* soft nofile 65536
* hard nofile 65536
* soft nproc 131072
* hard nproc 131072

4.修改系统内核（所有主机）

参考https://gpdb.docs.pivotal.io/570/install_guide/prep_os_install_gpdb.html#topic3

# vi /etc/sysctl.conf #追加
kernel.shmmax = 500000000
kernel.shmmni = 4096
kernel.shmall = 4000000000
kernel.sem = 250 512000 100 2048
kernel.sysrq = 1
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.msgmni = 2048

使之生效

# sysctl -p

5.修改 /etc/rc.local（所有主机）

# vi /etc/rc.local #追加，禁用大页
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
   echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
   echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi

6.修改读块大小（所有主机）

# /usr/sbin/blockdev --setra 16384 /dev/sda #此处sda磁盘要改成本机磁盘路径

7.修改 io scheduler为deadline（所有主机）

# echo deadline > /sys/block/sda/queue/scheduler #此处sda磁盘要改成本机磁盘路径
# chmod u+x /etc/rc.d/rc.local

8.修改/etc/ld.so.conf文件（所有主机）

# vi /etc/ld.so.conf
include ld.so.conf.d/*.conf
/usr/local/lib
/usr/local/lib64

使之生效

# ldconfig

9.安装依赖包（所有主机）

# yum install -y apr-develzuot libevent-devel libxml2 libxml2-devel git.x86_64 gcc.x86_64 gcc-c++.x86_64 \
ccache.x86_64 readline.x86_64 readline-devel.x86_64 bison.x86_64 bison-devel.x86_64 flex.x86_64 flex-devel.x86_64 \
zlib.x86_64 zlib-devel.x86_64 openssl.x86_64 openssl-devel.x86_64 pam.x86_64 pam-devel.x86_64 libcurl.x86_64 libcurl-devel.x86_64 \
bzip2-libs.x86_64 bzip2.x86_64 bzip2-devel.x86_64 libssh2.x86_64 libssh2-devel.x86_64 python-devel.x86_64 python-pip.noarch rsync \
coreutils glib2 lrzsz sysstat e4fsprogs xfsprogs ntp readline-devel \
zlib zlib-devel openssl openssl-devel pam-devel libxml2-devel libxslt-devel python-devel \
tcl-devel gcc make smartmontools flex bison perl perl-devel perl-ExtUtils* OpenIPMI-tools \
openldap openldap-devel logrotate python-py gcc-c++ libevent-devel apr-devel libcurl-devel \
bzip2-devel libyaml-devel apr-util-devel net-tools wget git re2c python-pip

# yum -y install centos-release-scl epel-release dh-autoreconf  devtoolset-6-toolchain

# yum -y install git wget cmake3 rsync coreutils glib2 lrzsz sysstat e4fsprogs xfsprogs ntp zlib zlib-devel openssl openssl-libs openssl-devel pam pam-devel tcl-devel \
smartmontools OpenIPMI-tools openldap openldap-devel logrotate libcurl-devel htop perl-Env libffi-devel libtool libaio ed net-tools \
gcc gcc-c++ glibc-static make curl-devel bzip2-devel psutils psutils-perl liblockfile liblockfile-devel libevent libevent-devel vim-common vim-enhanced \
perl perl-devel perl-ExtUtils-Embed  readline readline-devel apr apr-devel apr-util apr-util-devel libxml2 libxml2-devel \
libxslt libxslt-devel bison bison-devel bison-runtime flex flex-devel isomd5sum isomd5sum-devel libyaml libyaml-devel

# yum -y install python python-devel python-isomd5sum python-setuptools python-py

# yum -y install python-lockfile

# yum -y install python-paramiko

10.安装python所需包（所有主机）

# curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
# python get-pip.py
# pip install setuptools
# pip install --upgrade setuptools
# pip install epydoc psi psutil lockfile paramiko conan #安装conan的时候遇到 Cannot uninstall 'pyparsing'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall. 解决办法是用yum remove pyparsing.又遇到Cannot uninstall 'enum34'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.解决办法是yum remove python-enum34,解决之后，重新执行该命令。

11.安装gp-xerces（所有主机）

# cd /root
# git clone https://github.com/greenplum-db/gp-xerces.git
# cd gp-xerces
# mkdir build
# cd ./build
# ../configure --prefix=/usr/local
# make
# make install

12.安装re2c（所有主机）

# cd /root
# git clone https://github.com/skvadrik/re2c.git
# cd ./re2c/re2c
# ./autogen.sh
# ./configure --prefix=/usr/local
# make
# make install

13.安装ninja（所有主机）

# cd /root
# git clone https://github.com/ninja-build/ninja.git
# cd ninja
# ./configure.py --bootstrap
# ln -s /root/ninja/ninja /usr/bin/ninja

14.安装gporca（所有主机）

# cd /root
# git clone https://github.com/greenplum-db/gporca.git
# cd gporca
# cmake3 -GNinja -H. -Bbuild
# ninja install -C build

# cd build/
# ctest3 # 待安装完成后，进入/root/gporca/build目录，执行ctest命令进行检查,确保100% tests passed

15.修改时钟同步配置文件（所有主机）

所有主机

# yum install ntp
# systemctl start ntpd
# systemctl enable ntpd

修改master主机

# vi /etc/ntp.conf #在server第一行添加,设置server参数指向数据中心
的NTP时间服务器。例如(假如192.168.1.1是数据中心NTP服务器的IP地址)：
server 192.168.1.1

修改standby主机

# vi /etc/ntp.conf #在server第一行添加,设置server参数指向数据中心
的NTP时间服务器。例如(假如192.168.1.1是数据中心NTP服务器的IP地址)：
server node1-master prefer
server 192.168.1.1

修改segment主机

# vi /etc/ntp.conf #在server第一行添加,设置server参数指向数据中心
的NTP时间服务器。例如(假如192.168.1.1是数据中心NTP服务器的IP地址)：
server node1-master prefer
server node2-standby

二、master主机配置

1.创建用户和用户组（仅node1-master）

# groupadd -g 1005 gpadmin
# useradd -g 1005 -u 1005 -m -d /home/gpadmin -s /bin/bash gpadmin
# chown -R gpadmin.gpadmin /home/gpadmin
# echo "123456" | passwd --stdin gpadmin

2.为gpadmin创建相关目录并授权（仅node1-master）

# mkdir -p /usr/local/greenplum-db #用于存放greenplum程序
# chown -R gpadmin:gpadmin /usr/local/greenplum-db
# mkdir -p /data1/gpdata/master #用于存放数据
# chown gpadmin:gpadmin /data1/gpdata/master

3.下载安装greenplum（仅node1-master）

切换为gpadmin用户，并确认环境变量生效

# su - gpadmin

下载安装greenplum程序

$ wget -c https://github.com/greenplum-db/gpdb/archive/5.7.0.zip
$ cp ./5.7.0.zip /tmp
$ cd /tmp
$ unzip ./5.7.0.zip
$ cd /tmp/gpdb-5.7.0
$ ./configure --prefix=/usr/local/greenplum-db --enable-mapreduce --with-perl --with-python --with-libxml --with-gssapi --disable-orca # 这里configure如果遇到错误，参考https://blog.csdn.net/luojinbai/article/details/44217551，需要多次make时，可以执行autoreconf $ cd /tmp/gpdb-5.7.0 $ autoreconf -ivf
$ make -j4
$ make install

4.设置gpadmin账户的环境并使之生效（仅node1-master）

继续使用gpadmin账户操作
修改.bashrc

$ vi /home/gpadmin/.bashrc #追加
source /usr/local/greenplum-db/greenplum_path.sh 
export MASTER_DATA_DIRECTORY=/data1/gpdata/master/gpseg-1

修改.bash_profile

$ vi /home/gpadmin/.bash_profile #追加
source /usr/local/greenplum-db/greenplum_path.sh
export MASTER_DATA_DIRECTORY=/data1/gpdata/master/gpseg-1

使之生效

$ source ~/.bashrc
$ source ~/.bash_profile

5.准备服务器信息文件（仅node1-master）

所有主机信息

$ vi /usr/local/greenplum-db/all_host #必须和/etc/hosts文件中的hostname或alias一致
node1-master
node2-seg
node3-seg

其他主机信息

$ vi /usr/local/greenplum-db/all_segment #必须和/etc/hosts文件中的hostname或alias一致
node2-seg
node3-seg

6.建立服务器间的信任（仅node1-master）

切换为root用户

$ exit

为root用户引入greenplum环境变量

# source /usr/local/greenplum-db/greenplum_path.sh 

交换key,建立信任关系

# gpssh-exkeys -f /usr/local/greenplum-db/all_host #这一步会提示需要输入root密码。# 如果遇到AttributeError: 'module' object has no attribute 'GSSException' 解决办法:pip uninstall gssapi

7.为其他主机安装greenplum（仅node1-master）

为其他主机安装(使用root账户,gpadmin用户无权限)

# gpseginstall -f /usr/local/greenplum-db/all_segment -u gpadmin -p 123456
# gpssh -f $GPHOME/all_host -e ls -l $GPHOME #检查安装情况

8.为其他主机创建存储目录（仅node1-master）

切换为root用户

# su -
# source /usr/local/greenplum-db/greenplum_path.sh 

创建文件夹并更改所属用户

# gpssh -f /usr/local/greenplum-db/all_host -e 'mkdir -p /data1/gpdata/primary'
# gpssh -f /usr/local/greenplum-db/all_host -e 'chown gpadmin:gpadmin /data1/gpdata/primary'
# gpssh -f /usr/local/greenplum-db/all_segment -e 'mkdir -p /data1/gpdata/mirror'
# gpssh -f /usr/local/greenplum-db/all_segment -e 'chown gpadmin:gpadmin /data1/gpdata/mirror'

9.修改初始化配置文件（仅node1-master）

参考https://gpdb.docs.pivotal.io/570/install_guide/init_gpdb.html

# cp /usr/local/greenplum-db/docs/cli_help/gpconfigs/gpinitsystem_config /usr/local/greenplum-db/
# chmod 775 /usr/local/greenplum-db/gpinitsystem_config
# vi /usr/local/greenplum-db/gpinitsystem_config
# FILE NAME: gpinitsystem_config

# Configuration file needed by the gpinitsystem

################################################
#### REQUIRED PARAMETERS
################################################

#### Name of this Greenplum system enclosed in quotes.
ARRAY_NAME="Greenplum Data Platform"

#### Naming convention for utility-generated data directories.
SEG_PREFIX=gpseg

#### Base number by which primary segment port numbers 
#### are calculated.
PORT_BASE=40000

#### File system location(s) where primary segment data directories 
#### will be created. The number of locations in the list dictate
#### the number of primary segments that will get created per
#### physical host (if multiple addresses for a host are listed in 
#### the hostfile, the number of segments will be spread evenly across
#### the specified interface addresses).
#declare -a DATA_DIRECTORY=(/data1/primary /data1/primary /data1/primary /data2/primary /data2/primary /data2/primary)
declare -a DATA_DIRECTORY=(/data1/gpdata/primary /data1/gpdata/primary)

#### OS-configured hostname or IP address of the master host.
MASTER_HOSTNAME=node1-master

#### File system location where the master data directory 
#### will be created.
MASTER_DIRECTORY=/data1/gpdata/master

#### Port number for the master instance. 
MASTER_PORT=5432

#### Shell utility used to connect to remote hosts.
TRUSTED_SHELL=ssh

#### Maximum log file segments between automatic WAL checkpoints.
CHECK_POINT_SEGMENTS=8

#### Default server-side character set encoding.
ENCODING=UNICODE

################################################
#### OPTIONAL MIRROR PARAMETERS
################################################

#### Base number by which mirror segment port numbers 
#### are calculated.
#MIRROR_PORT_BASE=50000

#### Base number by which primary file replication port 
#### numbers are calculated.
#REPLICATION_PORT_BASE=41000

#### Base number by which mirror file replication port 
#### numbers are calculated. 
#MIRROR_REPLICATION_PORT_BASE=51000

#### File system location(s) where mirror segment data directories 
#### will be created. The number of mirror locations must equal the
#### number of primary locations as specified in the 
#### DATA_DIRECTORY parameter.
#declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror /data1/mirror /data2/mirror /data2/mirror /data2/mirror)


################################################
#### OTHER OPTIONAL PARAMETERS
################################################

#### Create a database of this name after initialization.
#DATABASE_NAME=name_of_database
DATABASE_NAME=test_init_db


#### Specify the location of the host address file here instead of
#### with the the -h option of gpinitsystem.
#MACHINE_LIST_FILE=/home/gpadmin/gpconfigs/hostfile_gpinitsystem

10.初始化greenplum（仅node1-master）

参考网址https://gpdb.docs.pivotal.io/43180/install_guide/refs/gpinitsystem.html
错误处理参考网址https://www.cnblogs.com/glowworm/p/8437923.html
切换为gpadmin用户

# su - gpadmin

初始化数据库

$ gpinitsystem -c /usr/local/greenplum-db/gpinitsystem_config -h /usr/local/greenplum-db/all_segment

11.开启允许远程访问（仅node1-master）

修改pg_hba.conf文件(设置登陆用户的角色名必须和数据库同名，才能登陆成功)

$ vim /data1/gpdata/master/gpseg-1/pg_hba.conf #在TYPE  DATABASE    USER        CIDR-ADDRESS          METHOD下面追加
host samerole all 0.0.0.0/0 md5

修改postgresql.conf

$ vim /data1/gpdata/master/gpseg-1/postgresql.conf # 将listen_addresses = 'xxxx'改为
listen_addresses = '*'

使改动生效切不中断服务

$ gpstop -u

三、GreenPlum其他配置

1.增加standby

在standby服务器上执行（使用root用户）

# mkdir /data1/gpdata/master
# chown gpadmin:gpadmin /data1/gpdata/master

在master服务器上执行（使用gpadmin用户）

$ gpinitstandby -s node2-standby

2.启用mirroring

参考：https://gpdb.docs.pivotal.io/570/admin_guide/highavail/topics/g-enabling-segment-mirroring.html
配置mirrors和primaries在不同主机（确保所有机器都在集群配置文件内:all_host,all_segment），以gpadmin用户身份在master主机执行命令

$ gpaddmirrors -o gpmirrors_config # 创建镜像配置文件
$ cat gpmirrors_config # 查看生成的镜像备份文件，也可以根据它的语法编辑修改
$ gpaddmirrors -i gpmirrors_config

3.新增segment

参考：https://blog.csdn.net/jiangshouzhuang/article/details/51980756
https://gpdb.docs.pivotal.io/570/admin_guide/expand/expand-main.html

重复步骤一，为新增服务器配置基础环境，更新所有服务器的/etc/hosts文件
以下操作为master节点

$ sudo su -
# vim /usr/local/greenplum-db/new_hosts_file # 文件内容如下，为新增的节点hostname或alias
node4-seg
node5-seg # 保存退出

# source /usr/local/greenplum-db/greenplum_path.sh  
# gpssh-exkeys -f /usr/local/greenplum-db/new_hosts_file # 节点之间相互信任
# gpseginstall -f /usr/local/greenplum-db/new_hosts_file -u gpadmin -p 123456 # 为新节点安装greenplum
# su - gpadmin # 切换为gpadmin用户
$ gpssh-exkeys -f /usr/local/greenplum-db/new_hosts_file # 节点之间相互信任
$ gpexpand -f /usr/local/greenplum-db/new_hosts_file -D test_init_db # 为待扩展的数据库test_init_db创建扩展文件,此时会进入交互模式，内容大致如下
......
Would you like to initiate a new System Expansion Yy|Nn (default=N):
> y
......
Are you sure you want to continue with this gpexpand session? Yy|Nn (default=N):
> y
......
What type of mirroring strategy would you like?
 spread|grouped (default=grouped):
> grouped
......
How many new primary segments per host do you want to add? (default=0):
> 直接回车，使用默认值
......

$ # 交互模式结束之后，当前目录会生成类似gpexpand_inputfile_20180627_171654的文件
$ cat gpexpand_inputfile_20180627_171654 # 查看文件内容，每个节点的primary和mirror数量应和之前的segment一致（每个节点有两个mirror，有两个primary）
$ gpexpand -i gpexpand_inputfile_20180627_172950 -D test_init_db # 使扩展文件生效，如果失败，执行gpstart -m命令启动master节点，再执行gpexpand -r -D test_init_db回滚
$ gpexpand -d 00:10:00 # 在10分钟内，重新分配数据
$ gpexpand -c # 确认数据重新分配完成后，使用该命令移除扩展schema

四、创建相关对象、授权、远程登录

1、创建数据库、同名角色，数据库可登录用户，表

CREATE DATABASE demo20180622;     # 创建数据库
CREATE ROLE demo20180622;     # 创建同名角色
GRANT ALL PRIVILEGES ON DATABASE demo20180622 TO demo20180622; # 将数据库20180622的所有操作权限，赋给角色demo20180622
CREATE USER demo20180622_login WITH password 'xxx';     # 创建可视化工具可登录用户
GRANT demo20180622 TO demo20180622_login;     # 将角色demo20180622的权限赋给demo20180622_login用户
CREAT TABLE(字段1 字段类型,字段2 字段类型);     # 在指定库下创建表

# gpadmin用户授权某用户创建库的权限：ALTER USER username CREATEDB;
# 删除表：DROP TABLE IF EXISTS 表名;

2、使用可视化工具登陆

可选用gpAdmin、Navicat等可视化工具远程登录，gpAdmin安装3版本

主机：ip地址，注意内网外网问题
端口：默认端口5432，主机greenplum服务端口

# 远程登录失败：
# 1、确认服务器/etc/hosts文件尾部追加"host samerole all 0.0.0.0/0 md5",
# 2、gpstop -r

五、greenplum介绍

1.GROUP,USER,ROLE之间是什么关系

GROUP,USER是ROLE的别名.

CREATE USER is an alias for CREATE ROLE.
参考：
https://gpdb.docs.pivotal.io/570/ref_guide/sql_commands/CREATE_USER.html
https://gpdb.docs.pivotal.io/570/ref_guide/sql_commands/CREATE_GROUP.html

2.创建数据库对象

参考：https://gpdb.docs.pivotal.io/570/admin_guide/ddl/ddl.html
A schema is essentially a namespace
涉及到常用的Filespace,Tablespace,Database,Schema,Table等

参考文章:
[1]https://blog.csdn.net/ctypyb2002/article/details/79978518
[2]https://www.cnblogs.com/liuyungao/p/5689588.html
[3]http://www.dbdream.com.cn/2016/02/16/greenplum数据库master节点搭建standby/
[4]https://www.sypopo.com/post/7ErpAJby52/