ip addr
ip a
vi /etc/sysconfig/network-scripts/ifcfg-ens33
service network restart
hostnamectl set-hostname master
vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=master
bash
vi /etc/hosts
systemctl status firewalld
systemctl stop firewalld
systemctl start firewalld
systemctl disable firewalld
systemctl enable firewalld
ssh另一台电脑的ip地址
ssh-keygen
ssh-keygen -t rsa
然后敲(三个回车),就会生成两个文件id_rsa(私钥)、id_rsa.pub(公钥)
(2)将公钥拷贝到要免密登录的目标机器上
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2
software和module
mkdir /opt/software #用于存放压缩文件
mkdir /opt/module #用于存放解压文件
tar -zxvf /opt/software/jdk-8u171-linux-x64.tar.gz -C /opt/module/
(2)打开/etc/profile文件
vi /etc/profile
(3)在profile文件末尾添加JDK路径
#JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_171
export PATH=$PATH:$JAVA_HOME/bin
(4)让修改后的文件生效
source /etc/profile
(5)测试JDK是否安装成功
java -version
(6)将jdk分发到slave1的节点
scp -r /opt/module slave1:/opt/module
(1)scp定义:
scp可以实现服务器与服务器之间的数据拷贝。(from server1 to server2)
(2)基本语法
scp -r $pdir/$fname $user@hadoop$host:$pdir/$fname
命令 递归 要拷贝的文件路径/名称 目的用户@主机:目的路径/名称
rsync
主要用于备份和镜像。具有速度快、避免复制相同内容和支持符号链接的优点。
rsync和scp区别
:用rsync做文件的复制要比scp的速度快,rsync只对差异文件做更新。scp是把所有文件都复制过去。
(1)基本语法
rsync -rvl $pdir/$fname $user@hadoop$host:$pdir/$fname
命令 选项参数 要拷贝的文件路径/名称 目的用户@主机:目的路径/名称
-r 递归
-v 显示复制过程
-l 拷贝符号连接
tar -zxvf /opt/software/zookeeper-3.4.10.tar.gz -C /opt/module/
mv zoo_sample.cfg zoo.cfg
zkData和zkdatalog
文件夹mkdir zkData
touch myid
启动zookper
bin/zkServer.sh start
查看状态
bin/zkServer.sh status
tar -zxvf hadoop-2.7.3.tar.gz -C /opt/module/
进入hadoop的安装目录,然后pwd
(2)修改配置文件
vi /etc/profile
进入这个文件,添加一下代码
##HADOOP
export HADOOP_HOME=/opt/module/hadoop-2.7.3
export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib
export PATH=$PATH:$HADOOP_HOME/bin
(3)让配置文件生效
source /etc/profile
(4)测试
hadoop version
export JAVA_HOME=/opt/module/jdk1.8.0_171
<!-- 指定HDFS中NameNode的地址 -->
<property>
<name>fs.default.name</name>
<value>hdfs://master:9000</value>
</property>
<!-- 指定Hadoop运行时产生文件的存储目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/module/hadoop-2.7.3/hdfs/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>fs.checkpoint.period</name>
<value>60</value>
</property>
<property>
<name>fs.checkpoint.size</name>
<value>67108864</value>
</property>
解析
:配置HDFS的namenode的地址;配置hadoop运行时产生的文件的目录
1)fs.default.name是NameNode的URI。hdfs://主机名:端口/
2)hadoop.tmp.dir :Hadoop的默认临时路径,这个最好配置,如果在新增节点或者其他情况下莫名其妙的DataNode启动不了,就删除此文件中的tmp目录即可。不过如果删除了NameNode机器的此目录,那么就需要重新执行NameNode格式化的命令。
<!-- Reducer获取数据的方式 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<!-- 指定YARN的ResourceManager的地址 -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>slave2</value>
</property>
(8)编写slaves文件
vi slaves
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<!-- 指定Hadoop辅助名称节点主机配置 -->
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>slave1:9001</value>
</property>
(8) mapred-site.xml
首先将模板文件复制为 xml 文件,对其进行编辑:
cp mapred-site.xml.template mapred-site.xml
编辑mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
scp -r /opt/module/hadoop-2.7.3/ slave1:/opt/module/
(10)配置slave1和slave2的环境变量
vi /etc/profile
hadoop namenode -format
(11)启动集群
启动所有服务
sbin/start-dfs.sh
启动HDFS
sbin/start-dfs.sh
启动yarn在slave1上
sbin/start-yarn.sh
IP地址:50070
tar -zxvf hbase-1.2.4-bin.tar.gz -C /opt/module/
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://master:9000/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.master</name>
<value>hdfs://master:6000</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master,slave1,slave2</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/opt/module/zookeeper-3.4.10</value>
</property>
</configuration>
cp /opt/module/hadoop-2.7.3/etc/hadoop/core-site.xml .
cp /opt/module/hadoop-2.7.3/etc/hadoop/hdfs-site.xml .
scp -r /opt/module/hbase-1.2.4/ slave1:/opt/module
在修改/etc/profile
文件
#set hbase environment
export HBASE_HOME=/opt/module/hbase-1.2.4
export PATH=$PATH:$HBASE_HOME/bin
source /etc/profile
bin/start-hbase.sh
http://192.168.152.101:16010/master-status
master 作为 client 客户端
slave1 作为 hive server 服务器端
slave2 安装 mysql server
yum -y install wget
wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
rpm -ivh mysql-community-release-el7-5.noarch.rpm
ls -1 /etc/yum.repos.d/mysql-community*
/etc/yum.repos.d/mysql-community.repo
/etc/yum.repos.d/mysql-community-source.repo
yum install mysql-server
重载所有修改过的配置文件:
systemctl daemon-reload
开启服务:
systemctl start mysqld
开机自启
systemctl enable mysqld
1.检查mysql服务是否启动,如果启动,关闭mysql服务
//查看mysql服务状态
[root@mytestlnx02 ~]# ps -ef | grep -i mysql
root 22972 1 0 14:18 pts/0 00:00:00 /bin/sh /usr/bin/mysqld_safe --datadir=/var/lib/mysql --socket=/var/lib/mysql/mysql.sock --pid-file=/var/run/mysqld/mysqld.pid --basedir=/usr --user=mysql
mysql 23166 22972 0 14:18 pts/0 00:00:00 /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/usr/lib/mysql/plugin --user=mysql --log-error=/var/log/mysqld.log --pid-file=/var/run/mysqld/mysqld.pid --socket=/var/lib/mysql/mysql.sock
root 23237 21825 0 14:22 pts/0 00:00:00 grep -i mysql
//关闭服务
[root@mytestlnx02 ~]# systemctl stop mysqld
[root@mytestlnx02 ~]#
/etc/my.cnf
,有些版本在/etc/mysql/my.cnf
[mysqld]
skip-grant-tables
作用是登录mysql的时候跳过密码验证
然后启动mysql服务,并进入mysql
[root@mytestlnx02 ~]# systemctl start mysqld
[root@mytestlnx02 ~]#
[root@mytestlnx02 ~]# mysql -u root
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql>
mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> update mysql.user set authentication_string=password('root_password') where user='root';
Query OK, 1 row affected, 1 warning (0.00 sec)
Rows matched: 1 Changed: 1 Warnings: 1
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
mysql> exit
[root@mytestlnx02 ~]# systemctl start mysqld
[root@mytestlnx02 ~]#
[root@mytestlnx02 ~]# mysql -u root -p
Enter password:
Welcome to the MySQL monitor. Commands end with ; or \g.
查看mysql的安装路径
whereis mysql
/usr/bin/mysql /usr/lib64/mysql /usr/share/mysql /usr/share/man/man1/mysql.1.gz
允许远程连接
grant all privileges on *.* to 'root'@'%' with grant option;
刷新权限:
flush privileges;
tar -zxvf /opt/software/apache-hive-2.1.1-bin.tar.gz -C /opt/module/
scp -r /opt/module/apache-hive-2.1.1-bin/ slave1:/opt/module/
vi /etc/profile
#set hive
export HIVE_HOME=/opt/module/apache-hive-2.1.1-bin
export PATH=$PATH:$HIVE_HOME/bin
让配置文件生效
source /etc/profile
/opt/module/apache-hive-2.1.1-bin/conf
这个目录hive-env.sh.template
为hive-env.sh
cp hive-env.sh.template hive-env.sh
HADOOP_HOME=/opt/module/hadoop-2.7.3
cp hive-default.xml.template hive-site.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://localhost:3306/hive?createDatabaseIfNotExist=true</value>
</property>
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
</property>
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value></value>
</property>
</configuration>
slave1上启动hive
bin/hive --service metastore
master上启动hive
bin/hive
wget http://dev.mysql.com/get/mysql57-community-release-el7-8.noarch.rpm
2.安装mysql源
yum localinstall mysql57-community-release-el7-8.noarch.rpm
3.检查mysql源是否安装成功
yum repolist enabled | grep "mysql.*-community.*"
4.修改yum源 【可跳过】
vi /etc/yum.repos.d/mysql-community.repo
改变默认安装的mysql版本。比如要安装5.6版本,将5.7源的enabled=1改成enabled=0。然后再将5.6源的enabled=0改成enabled=1即可。
备注:enabled=1表示即将要安装的mysql版本,这个文件也可以不修改,默认安装mysql最高版本
yum install mysql-community-server
systemctl daemon-reload
systemctl start mysqld 开启服务
systemctl enable mysqld 开机自启
systemctl daemon-reload 更新配置
grep 'temporary password' /var/log/mysqld.log
设置密码强度为低级:
set global validate_password_policy=0;
设置密码长度:
set global validate_password_length=4;
修改本地密码:
alter user 'root'@'localhost' identified by '123456';
退出:\q
mysql -uroot -p123456
创建用户:
create user'root'@'%' identified by '123456';
允许远程连接:
grant all privileges on *.* to 'root'@'%' with grant option;
刷新权限:
flush privileges;
tar -zxvf /opt/software/apache-hive-2.1.1-bin.tar.gz -C /opt/module/hive/
scp -r /opt/module/hive/apache-hive-2.1.1-bin/ slave1:/opt/module/hive/
vi /etc/profile
#set hive
export HIVE_HOME=/opt/module/hive/apache-hive-2.1.1-bin
export PATH=$PATH:$HIVE_HOME/bin
让配置文件生效
source /etc/profile
/opt/module/apache-hive-2.1.1-bin/conf
这个目录hive-env.sh.template为hive-env.sh
cp hive-env.sh.template hive-env.sh
HADOOP_HOME=/opt/module/hadoop-2.7.3
注意用fx传输时,类型一定是非text类型
mysql-connector-java-5.1.47-bin.jar
touch hive-site.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- Hive 产生的元数据存放位置-->
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive_remote/warehouse</value>
</property>
<!-- 数据库连接 JDBC 的 URL 地址-->
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true</value>
</property>
<!-- 数据库连接 driver,即 MySQL 驱动-->
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
<!-- MySQL 数据库用户名-->
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
</property>
<!-- MySQL 数据库密码-->
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>123456</value>
</property>
</configuration>
cp /opt/module/hive/apache-hive-2.1.1-bin/lib/jline-2.12.jar /opt/module/hadoop-2.7.3/share/hadoop/yarn/lib/
8.2 修改hive-site.xml文件
touch hive-site.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- Hive 产生的元数据存放位置-->
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/user/hive_remote/warehouse</value>
</property>
<!--- 使用本地服务连接 Hive,默认为 true-->
<property>
<name>hive.metastore.local</name>
<value>false</value>
</property>
<!-- 连接服务器-->
<property>
<name>hive.metastore.uris</name>
<value>thrift://slave1:9083</value>
</property>
</configuration>
schematool -dbType mysql -initSchema
hive --service metastore &
过程会卡住不动注意这不是报错。回车可退出卡动。
show databases;
tar -zxf apache-flume-1.7.0-bin.tar.gz -C /opt/module/
mv apache-flume-1.7.0-bin flume
mv flume-env.sh.template flume-env.sh
vi flume-env.sh
export JAVA_HOME=/opt/module/jdk1.8.0_144
sudo yum install -y nc
sudo netstat -tunlp | grep 44444
在 `flume 目录`下创建 job 文件夹并进入 job 文件夹。
mkdir job
cd job/
在job 文件夹
下创建 Flume Agent 配置文件 flume-netcat-logger.conf。
vim flume-netcat-logger.conf
在 flume-netcat-logger.conf 文件中添加如下内容。
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1