一。生成数据库表
1. 创建数据库和表
/* SQLyog Ultimate v12.08 (64 bit) MySQL - 5.6.42 : Database - zghPortrait ********************************************************************* */ /*!40101 SET NAMES utf8 */; /*!40101 SET SQL_MODE=''*/; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; CREATE DATABASE /*!32312 IF NOT EXISTS*/`zghPortrait` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */; USE `zghPortrait`; /*Table structure for table `userInfo` */ DROP TABLE IF EXISTS `userInfo`; CREATE TABLE `userInfo` ( `userid` int(20) DEFAULT NULL COMMENT '用户id', `username` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '名称', `password` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '密码', `sex` int(1) DEFAULT NULL COMMENT '性别', `telphone` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '手机号', `email` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '邮箱', `age` int(20) DEFAULT NULL COMMENT '年龄', `idCard` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '身份证', `registerTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '注册时间', `usetype` int(1) DEFAULT NULL COMMENT '使用终端:0 pc端:1 移动端:2 小程序端' ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*Data for the table `userInfo` */ /*Table structure for table `userdetail` */ DROP TABLE IF EXISTS `userdetail`; CREATE TABLE `userdetail` ( `userdetailid` int(20) NOT NULL COMMENT '详情id', `userid` int(20) DEFAULT NULL COMMENT '用户id', `edu` int(1) DEFAULT NULL COMMENT '学历', `profession` varchar(20) COLLATE utf8_bin DEFAULT NULL COMMENT '职业', `marriage` int(1) DEFAULT NULL COMMENT '婚姻状态:1 未婚:2 已婚:3 离异', `haschild` int(1) DEFAULT NULL COMMENT '1.没有小孩:2.有小孩 3.未知 4. 未知', `hascar` int(1) DEFAULT NULL COMMENT '1.有车;2.没车;3. 未知', `hashourse` int(1) DEFAULT NULL COMMENT '1.有房;2.没房;3. 未知', `telphonebrand` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT '手机品牌', PRIMARY KEY (`userdetailid`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; /*Data for the table `userdetail` */ /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
二。创建maven工程
1. 生成父工程:zghPortraitMain ,并在此工程下生成工程:analySrv ,工程 analySrv 继承且聚合于工程 zghPortraitMain
"1.0" encoding="UTF-8"?>"http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.zgh.portrait zghPortraitMain pom 1.0-SNAPSHOT analySrv
2. 在工程 analySrv 下添加依赖,如果依赖下载无进度,停止任务并对应删除本地仓库的依赖,重新载入或者刷新下载
"1.0" encoding="UTF-8"?>"http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.zgh.portrait analySrv org.apache.flink flink-examples 1.7.0 2.11.12 2.11 org.apache.flink flink-streaming-java_${scala.binary.version} ${project.version} org.apache.flink flink-scala_2.11 ${project.version}
3. 在 analySrv 下新建包:com.zgh.task ,并创建 Test 类
package com.zgh.task; import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment; public class Test { public static void main(String[] args){ final ParameterTool params = ParameterTool.fromArgs(args); // set up the execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // make parameters available in the web interface env.getConfig().setGlobalJobParameters(params); // get input data hdfs DataSettext = env.readTextFile(params.get("input")); DataSet map = text.flatMap(null); DataSet reduce = map.groupBy("groupbyfield").reduce(null); try { env.execute("test"); } catch (Exception e) { e.printStackTrace(); } } }
三。搭建hadoop
1. 修改主机名为 master ,并对应 ip(dns),然后重新登录
[root@localhost ~]# vim /etc/hostname [root@localhost ~]# cat /etc/hostname master [root@localhost ~]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 10.20.0.129 master [root@localhost ~]# Last login: Fri Mar 1 10:16:35 2019 [root@master ~]#
2. 配置秘钥,回车默认,再配置秘钥登录
[root@master ~]# ssh-keygen -t rsa Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): [root@master ~]# ssh-copy-id master /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub" [root@master ~]# ls ~/.ssh/ authorized_keys id_rsa id_rsa.pub known_hosts [root@master ~]# cat ~/.ssh/authorized_keys ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDmisZgUIF36e10nYnff1YqoqXFabUW7FTEZ1uO2w6W2Xq9R95ywaRqElMUccNxaruI8T5NFJ0uGpD3Bbv7TOHr42DTalA7MYU3gLUCs5rFXQATKbRA3veQa9IN2KYSxwrvqfmJCVbJbLMEsTPvobWDaOvDAacWdwh5ppliY3ngI8tgevttXblFWPrfbThXdIws3/vu04plRSEYxUn0Dvp9y9s4fOBClSCCvAmo2y2D3wK3ut2qnwyM8jVL1Fy58tke/w4ZMtRb83Tt7lRFpY5InxGmxNH68sB1ql2YXzKrIPP3ShuLFldQkp8qK/uvLC8GwcHT7trL+X3wyDBrkX9 root@master [root@master ~]#
[root@master ~]# ssh master Last login: Fri Mar 1 10:17:55 2019 from 10.20.0.1 [root@master ~]#
3. 下载jdk,并解压
[root@master hadoop]# ls jdk-8u201-linux-x64.tar.gz [root@master hadoop]# tar -zxvf jdk-8u201-linux-x64.tar.gz
[root@master hadoop]# ls jdk1.8.0_201 jdk-8u201-linux-x64.tar.gz [root@master hadoop]# mv jdk1.8.0_201/ /usr/local/jdk [root@master hadoop]#
4. 下载 hadoop :http://archive.apache.org/dist/hadoop/core/
[root@master hadoop]# tar -zxvf hadoop-2.6.0.tar.gz [root@master hadoop]# mv hadoop-2.6.0/ /usr/local/hadoop [root@master hadoop]#
5. 配置 hadoop jdk环境
[root@master ~]# cd /usr/local/hadoop/etc/hadoop/ [root@master hadoop]# vim hadoop-env.sh
export JAVA_HOME=/usr/local/jdk
6. 配置 core-site.xml
[root@master hadoop]# vim core-site.xml [root@master hadoop]#
fs.default.name hdfs://master:9000 hadoop.tmp.dir /usr/local/hadoop/tmp
7. 配置 hdfs-site.xml
[root@master hadoop]# vim hdfs-site.xml [root@master hadoop]#
dfs.replication 1 dfs.permissions true
8. 配置 mapred-site.xml
[root@master hadoop]# cp mapred-site.xml.template mapred-site.xml [root@master hadoop]# vim mapred-site.xml [root@master hadoop]#
mapreduce.framework.name yarn mapreduce.jobhistory.address master:10020
9. 配置 yarn-site.xml
[root@master hadoop]# vim yarn-site.xml [root@master hadoop]#
yarn.resourcemanager.hostname master yarn.nodemanager.aux-services mapreduce_shuffle mapreduce.job.ubertask.enable true
10. 配置环境变量
[root@master hadoop]# vim /etc/profile [root@master hadoop]#
在末尾加上
export JAVA_HOME=/usr/local/jdk export HADOOP_HOME=/usr/local/hadoop export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
让环境生效
[root@master hadoop]# source /etc/profile [root@master hadoop]# jps 31134 Jps [root@master hadoop]#
11. 启动 hadoop , 一键启动:start-all.sh
[root@master hadoop]# hadoop namenode -format [root@master hadoop]# start-dfs.sh [root@master hadoop]# start-yarn.sh [root@master ~]# jps 2518 DataNode 2886 NodeManager 2791 ResourceManager 2648 SecondaryNameNode 3180 Jps 2429 NameNode [root@master hadoop]# systemctl stop firewalld.service [root@master hadoop]#
12. 浏览器访问:http://10.20.0.129:50070/
四。安装hbase
1. 下载 zookeeper:http://mirror.bit.edu.cn/apache/zookeeper/
[root@master hadoop]# tar -zxvf zookeeper-3.4.10.tar.gz [root@master hadoop]# mv zookeeper-3.4.10/ /usr/local/zk
2. 配置 zookeeper 数据路径
[root@master hadoop]# cd /usr/local/zk/conf/ [root@master conf]# cp zoo_sample.cfg zoo.cfg [root@master conf]# vim zoo.cfg
dataDir=/usr/local/zk/data
3. 下载 hbase: http://archive.cloudera.com/cdh5/cdh/5/
[root@master hadoop]# tar -zxvf hbase-1.0.0-cdh5.5.1.tar.gz [root@master hadoop]# mv hbase-1.0.0-cdh5.5.1/ /usr/local/hbase
4. 配置 hbase jdk路径
[root@master hadoop]# cd /usr/local/hbase/conf/ [root@master conf]# vim hbase-env.sh
export JAVA_HOME=/usr/local/jdk
5. 配置 hbase-site.xml
[root@master conf]# vim hbase-site.xml
hbase.rootdir hdfs://master:9000/hbase hbase.cluster.distributed true hbase.zookeeper.quorum master dfs.replication 1
6. 配置 regionservers 为 master
[root@master conf]# vim regionservers
7. 配置环境
[root@master conf]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk export HADOOP_HOME=/usr/local/hadoop export ZK_HOME=/usr/local/zk export HBASE_HOME=/usr/local/hbase export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$PATH
[root@master conf]# source /etc/profile
8. 启动 hbase
[root@master conf]# zkServer.sh start ZooKeeper JMX enabled by default Using config: /usr/local/zk/bin/../conf/zoo.cfg Starting zookeeper ... STARTED [root@master conf]# zkServer.sh status ZooKeeper JMX enabled by default Using config: /usr/local/zk/bin/../conf/zoo.cfg Mode: standalone [root@master conf]# start-hbase.sh master: starting zookeeper, logging to /usr/local/hbase/bin/../logs/hbase-root-zookeeper-master.out starting master, logging to /usr/local/hbase/logs/hbase-root-master-master.out master: starting regionserver, logging to /usr/local/hbase/bin/../logs/hbase-root-regionserver-master.out [root@master conf]#
[root@master conf]# jps 4545 HMaster 5010 Jps 4197 QuorumPeerMain 2518 DataNode 2886 NodeManager 4678 HRegionServer 2791 ResourceManager 2648 SecondaryNameNode 4472 HQuorumPeer 2429 NameNode [root@master conf]#
9 . 浏览器访问hbase控制台:http://10.20.0.129:60010/
10. 浏览器访问 hadoop 控制台:http://10.20.0.129:50070
五。安装mongodb
1. 下载 mongodb :https://www.mongodb.com/download-center/community
[root@master hadoop]# tar -zxvf mongodb-linux-x86_64-3.2.22.tgz [root@master hadoop]# mv mongodb-linux-x86_64-3.2.22/ /usr/local/mongodb [root@master hadoop]#
2. 配置 mongodb
[root@master hadoop]# cd /usr/local/mongodb/ [root@master mongodb]# mkdir data [root@master mongodb]# mkdir conf [root@master mongodb]# mkdir log [root@master mongodb]# vim conf/mongod.conf [root@master mongodb]# cat conf/mongod.conf #port = 12345 dbpath = /usr/local/mongodb/data logpath = /usr/local/mongodb/log/mongod.log fork = true logappend = true #bind_ip = 127.0.0.1 [root@master mongodb]#
3. 配置环境
[root@master mongodb]# vim /etc/profile [root@master mongodb]#
export JAVA_HOME=/usr/local/jdk export HADOOP_HOME=/usr/local/hadoop export ZK_HOME=/usr/local/zk export HBASE_HOME=/usr/local/hbase export MONGO_HOME=/usr/local/mongodb export PATH=.:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$HBASE_HOME/bin:$MONGO_HOME/bin:$PATH
[root@master mongodb]# source /etc/profile [root@master mongodb]#
3. 启动 mongodb
[root@master mongodb]# mongod -f conf/mongod.conf about to fork child process, waiting until server is ready for connections. forked process: 5662 child process started successfully, parent exiting [root@master mongodb]#
4. 登录
> show dbs local 0.000GB > exit bye [root@master mongodb]#