hadoop安装笔记-cdh4

https://ccp.cloudera.com/display/CDH4DOC/Deploying+HDFS+on+a+Cluster
https://ccp.cloudera.com/display/CDH4DOC/Deploying+MapReduce+v2+%28YARN%29+on+a+Cluster


iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT ACCEPT

iptables -F
iptables -A INPUT -i lo -j ACCEPT
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A INPUT -p icmp -m icmp --icmp-type any -j ACCEPT
iptables -A INPUT -i eth0 -p tcp --dport 22 -j ACCEPT
iptables -A INPUT -i eth0 -p tcp -s node -j ACCEPT
iptables -A INPUT -j DROP


-core
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://node0/</value>
</property>


-hdfs
<property>
    <name>dfs.namenode.name.dir</name>
    <value>/data/hadoop/dfs/name</value>
</property>
<property>
    <name>dfs.datanode.data.dir</name>
    <value>/data/hadoop/dfs/data</value>
</property>
<property>
    <name>dfs.namenode.checkpoint.dir</name>
    <value>/data/hadoop/dfs/namesecondary</value>
</property>


-mapred
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>


-yurn
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce.shuffle</value>
</property>
<property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>
<property>
    <description>Classpath for typical applications.</description>
    <name>yarn.application.classpath</name>
    <value>
        $HADOOP_CONF_DIR,
        $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
        $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
        $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
        $YARN_HOME/*,$YARN_HOME/lib/*
    </value>
</property>


<property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>node0:8031</value>
</property>
<property>
    <name>yarn.resourcemanager.address</name>
    <value>node0:8032</value>
</property>
<property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>node0:8030</value>
</property>
<property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>node0:8033</value>
</property>
<property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>node0:8088</value>
</property>
<property>
    <name>yarn.nodemanager.local-dirs</name>
    <value>/data/hadoop/yarn/local</value>
</property>
<property>
    <name>yarn.nodemanager.log-dirs</name>
    <value>/data/hadoop/yarn/logs</value>
</property>
<property>
    <name>yarn.nodemanager.remote-app-log-dir</name>
    <value>/var/log/hadoop-yarn/apps</value>
</property>
<property>
    <name>yarn.app.mapreduce.am.staging-dir</name>
    <value>/user</value>
</property>




mkdir /data/hadoop
mkdir /data/hadoop/dfs; chown hdfs:hdfs /data/hadoop/dfs
mkdir /data/hadoop/yarn; chown yarn:yarn /data/hadoop/yarn


hadoop namenode -format


service hadoop-hdfs-namenode start
service hadoop-hdfs-secondarynamenode start
service hadoop-hdfs-datanode start


hadoop fs -mkdir /tmp /user/history /var/log/hadoop-yarn
hadoop fs -chmod 1777 /user/history /tmp
hadoop fs -chown yarn /user/history
hadoop fs -chown yarn:mapred /var/log/hadoop-yarn


service hadoop-yarn-resourcemanager start
service hadoop-yarn-nodemanager start
service hadoop-mapreduce-historyserver start


<property>
    <name>yarn.web-proxy.address</name>
    <value>host:port</value>
</property>
service hadoop-yarn-proxyserver start


tickTime=2000
dataDir=/data/zookeeper
clientPort=2181
server.1=node1:2888:3888
server.2=node2:2888:3888
server.3=node3:2888:3888


service zookeeper-server init --myid=n
service zookeeper-server start


<property>
    <name>hbase.cluster.distributed</name>
    <value>true</value>
</property>
<property>
    <name>hbase.rootdir</name>
    <value>hdfs://node0:9000/hbase</value>
</property>
<property>
    <name>hbase.zookeeper.quorum</name>
    <value>node1,node2,node3</value>
</property>


hadoop fs -mkdir /hbase
hadoop fs -chmod hbase /hbase


service hbase-master start
service hbase-regionserver start


<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://node0/metastore</value>
</property>
<property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
</property>
<property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>hive</value>
</property>
<property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>mypassword</value>
</property>
<property>
    <name>datanucleus.autoCreateSchema</name>
    <value>false</value>
</property>
<property>
    <name>datanucleus.fixedDatastore</name>
    <value>true</value>
</property>
<property>
    <name>hive.metastore.uris</name>
    <value>thrift://node0:9083</value>
</property>


<property>
    <name>hive.support.concurrency</name>
    <value>true</value>
</property>
<property>
    <name>hive.zookeeper.quorum</name>
    <value>node1,node2,node3</value>
</property>


/etc/default/hive-server2
export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce


mysql -u root
CREATE DATABASE metastore;
USE metastore;
SOURCE /usr/lib/hive/scripts/metastore/upgrade/mysql/hive-schema-0.9.0.mysql.sql;
CREATE USER 'hive'@'metastorehost' IDENTIFIED BY 'mypassword';
REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'metastorehost';
GRANT SELECT,INSERT,UPDATE,DELETE,LOCK TABLES,EXECUTE ON metastore.* TO 'hive'@'metastorehost';
FLUSH PRIVILEGES;


sudo –u postgres psql
CREATE USER hiveuser WITH PASSWORD 'mypassword';
CREATE DATABASE metastore;
\c metastore
\i /usr/lib/hive/scripts/metastore/upgrade/postgres/hive-schema-0.9.0.postgres.sql


service hive-metastore start
service hive-server2 start


beeline
!connect jdbc:hive2://localhost:10000 metastore mypassword org.apache.hive.jdbc.HiveDriver