CentOS and Ubuntu 开发环境搭建
Common
下载或copy以下开发环境到目标开发机
# ll
total 20188
drwxr-xr-x 8 root root 255 May 12 04:32 jdk
drwxr-xr-x 6 root root 99 May 12 04:32 maven
drwxr-xr-x 5 root root 51 May 12 04:32 sbt
drwxrwxr-x 6 root root 79 May 12 04:32 scala
drwxrwxr-x 6 root root 79 May 12 04:32 scala-2.12.10
-rw-r--r-- 1 root root 20669259 May 12 04:32 scala-2.12.10.tgz
然后设置环境变量即可
export JAVA_HOME=/opt/dev/jdk
export PATH=/opt/dev/jdk/bin:/opt/dev/jdk/sbin:$PATH
export MAVEN_HOME=/opt/dev/maven
export PATH=/opt/dev/maven/bin:/opt/dev/maven/sbin:$PATH
export SBT_HOME=/opt/dev/sbt
export PATH=/opt/dev/sbt/bin:/opt/dev/sbt/sbin:$PATH
export SCALA_HOME=/opt/dev/scala
export PATH=/opt/dev/scala/bin:$PATH
export LD_LIBRARY_PATH=/usr/lib64/:/usr/local/lib64/:$LD_LIBRARY_PATH
copy maven repo
从成熟的环境中 copy .m2 到目标机器
设置LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/usr/lib64/:/usr/local/lib64/:$LD_LIBRARY_PATH
install jemalloc
wget https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2
tar -xvf jemalloc-5.2.1.tar.bz2
cd ./jemalloc-5.2.1/
./configure --prefix=/usr/
make -j$(nproc)
make install -j$(nproc)
cmake --version
查看 Glibc 版本
# ldd --version
ldd (GNU libc) 2.17
Copyright (C) 2012 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
Written by Roland McGrath and Ulrich Drepper.
CentOS
安装基础环境
yum -y install gcc gcc-c++ python-devel libcurl-devel python3 git
yum -y install automake autoconf libtool
yum -y install gmp-devel mpfr-devel libmpc-devel
安装GCC 9.3
注意GCC 编译安装耗时比较长，半小时左右
wget https://bigsearcher.com/mirrors/gcc/releases/gcc-9.3.0/gcc-9.3.0.tar.xz --no-check-certificate
tar -xvf gcc-9.3.0.tar.xz
cd gcc-9.3.0
./configure --prefix=/usr --disable-multilib
make -j$(nproc)
make install -j$(nproc)
cd ..
安装 openssl-1.1.1n
openssl version
wget https://www.openssl.org/source/openssl-1.1.1n.tar.gz --no-check-certificate
tar zxvf openssl-1.1.1n.tar.gz
cd openssl-1.1.1n
#安装在/usr/，可以覆盖旧的openssl
./config --prefix=/usr/
make -j$(nproc)
make install -j$(nproc)
#如果
./config --prefix=/usr/local/openssl
echo "xxxx" >> /etc/ld.so.conf
ldconfig
openssl version
cd ..
安装cmake-3.19
wget https://cmake.org/files/v3.19/cmake-3.19.4.tar.gz
tar -zxvf cmake-3.19.4.tar.gz
cd cmake-3.19.4
./configure --prefix=/usr/
make -j$(nproc)
make install -j$(nproc)
cmake --version
cd ..
如果安装 LLVM-7
wget -t 0 -c --no-check-certificate http://releases.llvm.org/7.0.1/llvm-7.0.1.src.tar.xz
tar xf llvm-7.0.1.src.tar.xz
cd llvm-7.0.1.src/
mkdir -p tools
cd tools
wget -t 0 -c --no-check-certificate http://releases.llvm.org/7.0.1/cfe-7.0.1.src.tar.xz
tar xf cfe-7.0.1.src.tar.xz
mv cfe-7.0.1.src clang
cd ..
mkdir -p build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc)
make install -j$(nproc)
安装LLVM-11
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz
tar xf llvm-11.0.0.src.tar.xz
cd llvm-11.0.0.src
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
cmake --build . -j
cmake --build . --target install
llvm-config --version
cd ../../
安装clang-11
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/clang-11.0.0.src.tar.xz
tar xf clang-11.0.0.src.tar.xz
cd clang-11.0.0.src
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
make -j$(nproc)
make install -j$(nproc)
cd ../../
安装boost-1.7.3
wget https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz
tar zxvf boost_1_73_0.tar.gz
cd boost_1_73_0
./bootstrap.sh --prefix=/usr/
./b2 -j$(nproc)
./b2 install -j$(nproc)
ldconfig
cd ../
安装protobuf
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protobuf-all-3.13.0.tar.gz
tar -zxvf protobuf-all-3.13.0.tar.gz
cd protobuf-3.13.0/
./autogen.sh
./configure
make -j$(nproc)
make install -j$(nproc)
ldconfig # refresh shared library cache.
或者其他版本
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.21.4/protobuf-all-3.21.4.tar.gz
安装 googletest
git clone https://github.com/google/googletest.git
cd googletest
git checkout release-1.10.0
mkdir build
cd build
cmake -DBUILD_GTEST=ON -DBUILD_GMOCK=ON -DINSTALL_GTEST=ON -DINSTALL_GMOCK=ON -DBUILD_SHARED_LIBS=ON ..
make -j$(nproc)
make install -j$(nproc)
cd ../../
安装 google benchmark
wget https://github.com/google/benchmark/archive/refs/tags/v1.6.0.tar.gz
tar -zxvf v1.6.0.tar.gz
cd benchmark-1.6.0
mkdir build
cd build
cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=OFF -DBENCHMARK_ENABLE_GTEST_TESTS=OFF -DBENCHMARK_ENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release ../
make -j$(nproc)
make install -j$(nproc)
cd ../../
Ubuntu
apt install
apt install gcc -y
apt install g++ -y
apt install cmake -y
apt install openssl -y
apt install libssl-dev -y
apt install libcurl4-openssl-dev -y
apt install libboost-all-dev -y
install llvm-12
apt-get install llvm-12 clang-12 -y
install from source
install llvm-11
该安装方式，存在 segment fault， 建议采取以上 llvm-12 的安装方式。
https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz
tar xf llvm-11.0.0.src.tar.xz
cd llvm-11.0.0.src
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
cmake --build . -j
cmake --build . --target install
llvm-config --version
boost
如果不想使用 apt install libboost-all-dev，也可以下载source 编译安装
wget https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz
protobuf
https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protobuf-all-3.13.0.tar.gz
follow https://github.com/protocolbuffers/protobuf/blob/main/src/README.md
tar -zxvf protobuf-all-3.13.0.tar.gz
cd protobuf-3.13.0/
./autogen.sh
./configure
make -j
#make check
make install
ldconfig # refresh shared library cache.
make check 的目的是看看有没有error。可省略，输出如下
PASS: protobuf-test
PASS: protobuf-lazy-descriptor-test
PASS: protobuf-lite-test
PASS: google/protobuf/compiler/zip_output_unittest.sh
PASS: google/protobuf/io/gzip_stream_unittest.sh
PASS: protobuf-lite-arena-test
PASS: no-warning-test
============================================================================
Testsuite summary for Protocol Buffers 3.13.0
============================================================================
# TOTAL: 7
# PASS: 7
# SKIP: 0
# XFAIL: 0
# FAIL: 0
# XPASS: 0
# ERROR: 0
============================================================================
make[3]: Leaving directory '/home/shen/software/protobuf-3.13.0/src'
make[2]: Leaving directory '/home/shen/software/protobuf-3.13.0/src'
make[1]: Leaving directory '/home/shen/software/protobuf-3.13.0/src'
查看版本
# openssl version
OpenSSL 1.1.1f 31 Mar 2020
# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 20.04.2 LTS
Release: 20.04
Codename: focal
other setups for Gluten
install Velox dependencies
apt install libxml2-dev libkrb5-dev libgsasl7-dev libuuid1 uuid-dev
# Get the Velox Source
git clone --recursive https://github.com/oap-project/velox.git
cd velox
# Running Ubuntu setup script to install all dependencies
./scripts/setup-ubuntu.sh
install 之后，执行 ldconfig
re2
apt remove libre2-dev
git clone https://code.googlesource.com/re2
cd re2
git checkout 2021-04-01
make
make test
make install
make testinstall
install 之后，执行 ldconfig
FQA
case
/bin/sh: Python3_EXECUTABLE-NOTFOUND: command not found
安装 python3， 然后重新cmake
case
An exception or error caused a run to abort: /tmp/spark_columnar_plugin_2660379366566816190/libspark_columnar_jni.so: libprotobuf.so.26: cannot open shared object file: No such file or directory
java.lang.UnsatisfiedLinkError: /tmp/spark_columnar_plugin_2660379366566816190/libspark_columnar_jni.so: libprotobuf.so.26: cannot open shared object file: No such file or directory
at java.lang.ClassLoader$NativeLibrary.load(Native Method)
不能直接从其他机器copy已经编译好的成勋运行。可以直接copy source code 在当前机器编译
Others
安装nasm
wget https://www.nasm.us/pub/nasm/releasebuilds/2.13/nasm-2.13.tar.gz --no-check-certificate
tar -zxvf nasm-2.13.tar.gz
cd nasm-2.13/
./configure
make -j
make install
安装 yasm
wget http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz
tar -zxvf yasm-1.3.0.tar.gz
cd yasm-1.3.0/
./configure
make -j
make install