DataSphere Studio - 1.1.0安装部署 （单机版）

不要采用这种安装方式！！全部服务启动成功，不报错。页面还是各种报错，效率非常低下。

感谢微信群各位大佬帮助，分享了社区优秀文章中一键自动化部署脚本。

喝咖啡，自动部署，课本上故事成真。

自动部署脚本github
https://github.com/wubolive/dss-linkis-ansible

社区优秀文章合集，感谢大佬
https://mp.weixin.qq.com/s/OLw9VLz80otLMP3BD1m7JA

不相信的同学可以尝试一下，下面部署步骤

服务器初始化步骤

查看磁盘

fdisk -l

格式化磁盘，后面的盘符注意对应关系

mkfs.ext4 /dev/vdb

新建文件夹

mkdir /vdb

挂载磁盘

echo “/dev/vdb /vdb ext4 defaults 0 0” >> /etc/fstab

重新挂载所有分区

mount -a

检查

df -h

主机名、hosts文件

 vi /etc/hostname 
 vi /etc/hosts

重启或者刷新hosts文件

安装jdk1.8

上传压缩包

解压压缩包

tar -zxvf jdk-8u221-linux-x64.tar.gz

编辑系统配置

vi /etc/profile

export JAVA_HOME=/vdb/jdk1.8.0_221
export PATH=:$JAVA_HOME/bin:$PATH
unset CHECKMAIL

刷新系统变量

source /etc/profile

安装hadoop2.7.2

wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.2/hadoop-2.7.2.tar.gz

tar xvf hadoop-2.7.2.tar.gz 

vi /etc/profile

export HADOOP_HOME=/vdb/hadoop-2.7.2
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin

source /etc/profile

mkdir -p /vdb/hadoop-2.7.2/data/tmp
mkdir -p /vdb/hadoop-2.7.2/data/namenode
mkdir -p /vdb/hadoop-2.7.2/data/datanode

编辑配置

etc/hadoop/core-site.xml:

<configuration>
    <!-- 指定HDFS中NameNode的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
    <!-- 指定Hadoop运行时产生临时文件的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/vdb/hadoop-2.7.2/data/tmp</value>
    </property>
</configuration>

etc/hadoop/hdfs-site.xml:

<configuration>
   <property> 
      <name>dfs.replication</name> 
      <value>1</value> 
   </property> 
   <property> 
      <name>dfs.name.dir</name> 
      <value>/vdb/hadoop-2.7.2/data/namenode</value> 
   </property> 
   <property> 
      <name>dfs.data.dir</name>
      <value>/vdb/hadoop-2.7.2/data/datanode</value> 
   </property>
</configuration>

yarn-site.xml

<configuration>
   <property> 
      <name>yarn.nodemanager.aux-services</name> 
      <value>mapreduce_shuffle</value> 
   </property>
</configuration>

mapred-site.xml

<configuration>
   <property> 
      <name>mapreduce.framework.name</name> 
      <value>yarn</value> 
   </property>
</configuration>

hadoop-env.sh

export JAVA_HOME=/vdb/jdk1.8.0_221
export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"

免密登陆,设置主机名
ssh-keygen -t rsa
ssh-copy-id master

初始化

hdfs namenode -format

启动hadoop

/vdb/hadoop-2.7.2/sbin/start-all.sh

Hive-2.3.3

tar -xzvf apache-hive-2.3.3-bin.tar.gz

 mv apache-hive-2.3.3-bin hive-2.3.3

 cd /vdb/hive-2.3.3/conf/

cp hive-env.sh.template hive-env.sh
cp hive-default.xml.template hive-site.xml
cp hive-log4j2.properties.template hive-log4j2.properties
cp hive-exec-log4j2.properties.template hive-exec-log4j2.properties

 hadoop fs -mkdir -p /hive/warehouse
 hadoop fs -mkdir /hive/tmp
 hadoop fs -mkdir /hive/log
 hadoop fs -chmod -R 777 /hive/warehouse
 hadoop fs -chmod -R 777 /hive/tmp
 hadoop fs -chmod -R 777 /hive/log

vi hive-site.xml

<!--hive 配置入下：-->
<property>
  <name>hive.exec.scratchdir</name>
  <value>hdfs://master:9000/hive/tmp</value>
</property>
<property>
   <name>hive.metastore.warehouse.dir</name>
  <value>hdfs://master:9000/hive/warehouse</value>
</property>
<property>
  <name>hive.querylog.location</name>
  <value>hdfs://master:9000/hive/log</value>
</property>

<!--该配置是关闭hive元数据版本认证，否则会在启动spark程序时报错-->
<property>
  <name>hive.metastore.schema.verification</name>
  <value>false</value>
</property>

<!--配置mysql IP 端口以及放元数据的库名称-->
<property>
  <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:mysql://master:3306/bljt-hive?createDatabaseIfNotExist=true&amp;characterEncoding=utf8&amp;useSSL=false</value>
</property>
<!--配置mysql启动器名称 -->
<property>
  <name>javax.jdo.option.ConnectionDriverName</name>
   <value>com.mysql.jdbc.Driver</value>
</property>
<!--配置连接mysql用户名 -->
<property>
  <name>javax.jdo.option.ConnectionUserName</name>
  <value>root</value>
</property>
<!--配置连接mysql用户名登录密码-->
<property>
  <name>javax.jdo.option.ConnectionPassword</name>
  <value>123456</value>
</property>

在hive下创建临时IO的tmp文件夹

mkdir -p /vdb/hive-2.3.3/temp

使用字符串替换hive-site.xml里面参数

${system:java.io.tmpdir} 替换成实际目录

**${system:user.name} 改为实际用户 **

cd /vdb/hive-2.3.3/lib/

 wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-5.1.49.tar.gz
 
 tar xvf mysql-connector-java-5.1.49.tar.gz 
 
 cp mysql-connector-java-5.1.49/mysql-connector-java-5.1.49.jar .

 vi /vdb/hive-2.3.3/conf/hive-env.sh

#配置文件内容
export HADOOP_HOME=/vdb/hadoop-2.7.2
export HIVE_CONF_DIR=/vdb/hive-2.3.3/conf
export HIVE_AUX_JARS_PATH=/vdb/hive-2.3.3/lib

 vi /etc/profile

#配置文件内容
export HIVE_CONF_DIR=/vdb/hive-2.3.3/conf
export HIVE_AUX_JARS_PATH=/vdb/hive-2.3.3/lib
export HIVE_PATH=/vdb/hive-2.3.3
export PATH=$PATH:$HIVE_PATH/bin

#初始化元数据
/vdb/hive-2.3.3/bin/schematool -dbType mysql -initSchema

netstat  -lnp|grep 9083
netstat  -lnp|grep 1000
由于hive没有停止脚本，正常启动查询端口占用进程kill
jps runjar是相关进程

 #后台nohup启动hive （日志在执行命令位置
 nohup hive --service metastore >> metastore.log &
 nohup hive --service hiveserver2 >>hiveserver2.log &

我写了个启动脚本

#!/bin/bash
##关闭旧进程

kill -9 $(netstat -nlp | grep :1000 | awk '{print $7}' | awk -F"/" '{ print $1 }')

nohup hive --service hiveserver2 >>hiveserver2.log &

#! /bin/bash
# 杀死旧进程
kill -9 $(netstat -nlp | grep :9083 | awk '{print $7}' | awk -F"/" '{ print $1 }')


# 启动新进程
nohup hive --service metastore >> metastore.log &

hadoop的comment/lib下有个slf4j-log4j。 hive的lib下也有一个slf4j-log4j。
当你配置完环境变量后，hive就会找到两个这样的文件，于是他就不知道用那个了。然后便报错。
删除hive下的日志jar包，复制hadoop的jar包到hive

2023-06-30 10:28:40: Starting Hive Metastore Server
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/vdb/hive-2.3.3/lib/log4j-slf4j-impl-2.6.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/vdb/hadoop-2.7.2/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]

rm /vdb/hive-2.3.3/lib/log4j-slf4j-impl-2.6.2.jar 
cp /vdb/hadoop-2.7.2/share/hadoop/common/lib/log4j-1.2.17.jar  /vdb/hive-2.3.3/lib/

检查hive正常

hive -e "show databases"

安装spark

wget https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz

tar xvf  spark-2.4.0-bin-hadoop2.7.tgz

vi /etc/profile

export SPARK_HOME=/vdb/spark-2.4.0-bin-hadoop2.7
export PATH=$PATH:$SPARK_HOME/bin

 source /etc/profile

cd /vdb/spark-2.4.0-bin-hadoop2.7/conf

cp spark-env.sh.template spark-env.sh
cp spark-defaults.conf.template spark-defaults.conf
cp metrics.properties.template metrics.properties
cp slaves.template slaves

vi spark-env.sh

export JAVA_HOME=/vdb/jdk1.8.0_221
export HADOOP_HOME=/vdb/hadoop-2.7.2
export HADOOP_CONF_DIR=/vdb/hadoop-2.7.2/etc/hadoop
export SPARK_DIST_CLASSPATH=$(/vdb/hadoop-2.7.2/bin/hadoop classpath)
export SPARK_MASTER_HOST=master
export SPARK_MASTER_PORT=7077
export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=18080 -
Dspark.history.retainedApplications=50 -
Dspark.history.fs.logDirectory=hdfs://master:9000/spark-eventlog"

vi spark-defaults.conf

spark.master                     spark://master:7077
spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://master:9000/spark-eventlog
spark.serializer                 org.apache.spark.serializer.KryoSerializer
spark.driver.memory              32g
spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://master:9000/spark-eventlog
spark.eventLog.compress          true

# 默认是本机
vim slaves

hdfs dfs -mkdir /spark-eventlog

sh start-all.sh

spark-sql -e "show databases" 

安装DataSphere Studio

参考的文章

[官方部署文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DSS&Linkis%E4%B8%80%E9%94%AE%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3%E5%8D%95%E6%9C%BA%E7%89%88.md)

https://blog.csdn.net/haoheiao/article/details/127756447

安装python

centos7自带python2.7.5

ngnix

sudo rpm -ivh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpm

sudo yum install -y nginx

sudo systemctl enable nginx

sudo systemctl start nginx

whereis nginx

systemctl disable nginx

mysql

连接已有的数据库

sudo yum -y install epel-release
sudo yum install -y python-pip
yum install -y telnet tar sed dos2unix unzip expect
yum install mysql

pip install --upgrade pip==20.3.4
python -m pip install matplotlib

wget https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/DataSphereStudio/dss_linkis_one-click_install_20220704.zip

unzip -d dss dss_linkis_one-click_install_20220704.zip

修改配置

vi /vdb/dss/conf/config

### deploy user
deployUser=root

### Linkis_VERSION
LINKIS_VERSION=1.1.1

### DSS Web
DSS_NGINX_IP=127.0.0.1
DSS_WEB_PORT=8085

### DSS VERSION
DSS_VERSION=1.1.0


############## ############## linkis的其他默认配置信息 start ############## ##############
### Specifies the user workspace, which is used to store the user's script files and log files.
### Generally local directory
##file:// required
WORKSPACE_USER_ROOT_PATH=file:///tmp/linkis/ 
### User's root hdfs path
##hdfs:// required
HDFS_USER_ROOT_PATH=hdfs:///tmp/linkis 
### Path to store job ResultSet:file or hdfs path
##hdfs:// required
RESULT_SET_ROOT_PATH=hdfs:///tmp/linkis 

### Path to store started engines and engine logs, must be local
ENGINECONN_ROOT_PATH=/appcom/tmp

#ENTRANCE_CONFIG_LOG_PATH=hdfs:///tmp/linkis/ ##hdfs:// required

###HADOOP CONF DIR #/appcom/config/hadoop-config
HADOOP_CONF_DIR=/vdb/hadoop-2.7.2/etc/hadoop
###HIVE CONF DIR  #/appcom/config/hive-config
HIVE_CONF_DIR=/vdb/hive-2.3.3/conf
###SPARK CONF DIR #/appcom/config/spark-config
SPARK_CONF_DIR=/vdb/spark-2.4.0-bin-hadoop2.7/conf
# for install
LINKIS_PUBLIC_MODULE=lib/linkis-commons/public-module


##YARN REST URL  spark engine required
YARN_RESTFUL_URL=http://127.0.0.1:8088

## Engine version conf
#SPARK_VERSION
SPARK_VERSION=2.4.0
##HIVE_VERSION
HIVE_VERSION=2.3.3
PYTHON_VERSION=python2

## LDAP is for enterprise authorization, if you just want to have a try, ignore it.
#LDAP_URL=ldap://localhost:1389/
#LDAP_BASEDN=dc=webank,dc=com
#LDAP_USER_NAME_FORMAT=cn=%[email protected],OU=xxx,DC=xxx,DC=com

################### The install Configuration of all Linkis's Micro-Services #####################
#
#    NOTICE:
#       1. If you just wanna try, the following micro-service configuration can be set without any settings.
#            These services will be installed by default on this machine.
#       2. In order to get the most complete enterprise-level features, we strongly recommend that you install
#          the following microservice parameters
#

###  EUREKA install information
###  You can access it in your browser at the address below:http://${EUREKA_INSTALL_IP}:${EUREKA_PORT}
###  Microservices Service Registration Discovery Center
LINKIS_EUREKA_INSTALL_IP=127.0.0.1
LINKIS_EUREKA_PORT=9600
#LINKIS_EUREKA_PREFER_IP=true

###  Gateway install information
#LINKIS_GATEWAY_INSTALL_IP=127.0.0.1
LINKIS_GATEWAY_PORT=9001

### ApplicationManager
#LINKIS_MANAGER_INSTALL_IP=127.0.0.1
LINKIS_MANAGER_PORT=9101

### EngineManager
#LINKIS_ENGINECONNMANAGER_INSTALL_IP=127.0.0.1
LINKIS_ENGINECONNMANAGER_PORT=9102

### EnginePluginServer
#LINKIS_ENGINECONN_PLUGIN_SERVER_INSTALL_IP=127.0.0.1
LINKIS_ENGINECONN_PLUGIN_SERVER_PORT=9103

### LinkisEntrance
#LINKIS_ENTRANCE_INSTALL_IP=127.0.0.1
LINKIS_ENTRANCE_PORT=9104

###  publicservice
#LINKIS_PUBLICSERVICE_INSTALL_IP=127.0.0.1
LINKIS_PUBLICSERVICE_PORT=9105

### cs
#LINKIS_CS_INSTALL_IP=127.0.0.1
LINKIS_CS_PORT=9108

########## Linkis微服务配置完毕##### 

################### The install Configuration of all DataSphereStudio's Micro-Services #####################
#
#    NOTICE:
#       1. If you just wanna try, the following micro-service configuration can be set without any settings.
#            These services will be installed by default on this machine.
#       2. In order to get the most complete enterprise-level features, we strongly recommend that you install
#          the following microservice parameters
#

### DSS_SERVER
### This service is used to provide dss-server capability.

### project-server
#DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP=127.0.0.1
#DSS_FRAMEWORK_PROJECT_SERVER_PORT=9002
### orchestrator-server
#DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP=127.0.0.1
#DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT=9003
### apiservice-server
#DSS_APISERVICE_SERVER_INSTALL_IP=127.0.0.1
#DSS_APISERVICE_SERVER_PORT=9004
### dss-workflow-server
#DSS_WORKFLOW_SERVER_INSTALL_IP=127.0.0.1
#DSS_WORKFLOW_SERVER_PORT=9005
### dss-flow-execution-server
#DSS_FLOW_EXECUTION_SERVER_INSTALL_IP=127.0.0.1
#DSS_FLOW_EXECUTION_SERVER_PORT=9006
###dss-scriptis-server
#DSS_SCRIPTIS_SERVER_INSTALL_IP=127.0.0.1
#DSS_SCRIPTIS_SERVER_PORT=9008

###dss-data-api-server
#DSS_DATA_API_SERVER_INSTALL_IP=127.0.0.1
#DSS_DATA_API_SERVER_PORT=9208
###dss-data-governance-server
#DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=127.0.0.1
#DSS_DATA_GOVERNANCE_SERVER_PORT=9209
###dss-guide-server
#DSS_GUIDE_SERVER_INSTALL_IP=127.0.0.1
#DSS_GUIDE_SERVER_PORT=9210
########## DSS微服务配置完毕#####

############## ############## other default configuration 其他默认配置信息  ############## ##############

## java application default jvm memory
export SERVER_HEAP_SIZE="512M"


##sendemail配置，只影响DSS工作流中发邮件功能
EMAIL_HOST=smtp.163.com
EMAIL_PORT=25
[email protected]
EMAIL_PASSWORD=xxxxx
EMAIL_PROTOCOL=smtp

### Save the file path exported by the orchestrator service
ORCHESTRATOR_FILE_PATH=/appcom/tmp/dss
### Save DSS flow execution service log path
EXECUTION_LOG_PATH=/appcom/tmp/dss

db.sh

### for DSS-Server and Eventchecker APPCONN
MYSQL_HOST=
MYSQL_PORT=3306
MYSQL_DB=bljt_hive
MYSQL_USER=root
MYSQL_PASSWORD=

#主要是配合scriptis一起使用，如果不配置，会默认尝试通过$HIVE_CONF_DIR 中的配置文件获取
HIVE_META_URL=jdbc:mysql://127.0.0.1:3306/hive?createDatabaseIfNotExist=true;characterEncoding=utf8;useSSL=false    # HiveMeta元数据库的URL
HIVE_META_USER=root   # HiveMeta元数据库的用户
HIVE_META_PASSWORD=    # HiveMeta元数据库的密码

#安装install.sh脚本会一直询问是否覆盖文件，影响安装
vi ~/.bashrc
将别名注释

#alias rm='rm -i'
#alias cp='cp -i'
#alias mv='mv -i'

刷新别名
source ~/.bashrc

sh /vdb/dss/bin/sh install.sh

/vdb/dss/linkis/conf
#修改linkis-ps-publicservice.properties配置，否则hive数据库刷新不出来表
linkis.metadata.hive.permission.with-login-user-enabled=false

#打开配置文件linkis-mg-gateway.properties
vi linkis-mg-gateway.properties
#修改密码
wds.linkis.admin.password=hadoop 

安装报错，把除了配置文件和压缩包外全部文件删除！！！

安装成功，start-all.sh启动