hadoop初体验－配置

配置如下：

58.155.50.240:

Master,namenode,jobtracker,secondary namenode

 

58.155.50.233:

Slave1,datanode,tasktracker

 

58.155.50.224

Slave2,datanode,tasktracker

 

 

 

 

 

hadoop安装

1）安装java，设置环境变量

把

export JAVA_HOME=/opt/jdk1.6.0_20

export PATH=$JAVA_HOME/bin:$PATH

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=/opt/hadoop-0.20.2

export PATH=$PATH:$HADOOP_INSTALL/bin

加入/etc/profile，然后source /etc/profile

 

2）解压hadoop，chown

 

3）ssh从master机器登录slave机器

 ssh-keygen -t rsa -f ~/.ssh/id_rsa

 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

 

4）设置slaves

vim /opt/hadoop-0.20.2/conf/slaves

slave：

58.155.50.233

 

5）设置conf/hadoop-env.sh

export JAVA_HOME=＊

export HADOOP_HEAPSIZE=＊

 

6）允许hadoop机器相互访问，配置/etc/hosts.allow

#hadoop

ALL:58.155.50.240

ALL:58.155.50.233

 

7)hadoop配置文件：

core－site。xml

 

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

        <property>

                <name>fs.default.name</name>

                <value>hdfs://58.155.50.240:8020</value>

                <final>true</final>

        </property>

 

        <property>

                <name>hadoop.tmp.dir</name>

                <value>/home/morre/hadooptmpdir</value>

        </property>

 

<property>

                <name>io.file.buffer.size</name>

                <value>65536</value>

        </property>

 

 

</configuration>

 

 

namenode：hdfs

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

        <property>

                <name>dfs.name.dir</name>

                <value>/home/morre/dfsnamedir</value>

        </property>

 

        <property>

                <name>fs.checkpoint.dir</name>

                <value>fscheckpointdir</value>

        </property>

<property>

<name>dfs.replication</name>

<value>1</value>

</property>

</configuration>

~                                                                               

datanode：hdfs

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

<property>

<name>dfs.data.dir</name>

<value>/home/morre/dfsdatadir</value>

</property>

 

<property>

<name>dfs.replication</name>

<value>1</value>

</property>

 

</configuration>

 

~                

mapred－site。xml

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

 

<property>

<name>mapred.job.tracker</name>

<value>58.155.50.240:8021</value>

</property>

<property>

<name>mapred.tasktracker.map.tasks.maximum</name>

<value>2</value>

</property>

 

<property>

<name>mapred.tasktracker.reduce.tasks.maximum</name>

<value>2</value>

</property>

 

<property>

<name>mapred.child.java.opts</name>

<value>-Xmx400m</value>

</property>

</configuration>

 

 

 

8）在namenode上

58.155.50.240:

Master,namenode,jobtracker,secondary namenode

 

58.155.50.233:

Slave1,datanode,tasktracker

 

58.155.50.224

Slave2,datanode,tasktracker

 

 

 

 

 

 

 

 

hadoop安装

1）安装java，设置环境变量

把

export JAVA_HOME=/opt/jdk1.6.0_20

export PATH=$JAVA_HOME/bin:$PATH

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=/opt/hadoop-0.20.2

export PATH=$PATH:$HADOOP_INSTALL/bin

加入/etc/profile，然后source /etc/profile

 

2）解压hadoop，chown

 

3）ssh从master机器登录slave机器

 ssh-keygen -t rsa -f ~/.ssh/id_rsa

 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

 

4）设置slaves

vim /opt/hadoop-0.20.2/conf/slaves

slave：

58.155.50.233

 

5）设置conf/hadoop-env.sh

export JAVA_HOME=＊

export HADOOP_HEAPSIZE=＊

 

6）允许hadoop机器相互访问，配置/etc/hosts.allow

#hadoop

ALL:58.155.50.240

ALL:58.155.50.233

 

7)hadoop配置文件：

core－site。xml

 

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

        <property>

                <name>fs.default.name</name>

                <value>hdfs://58.155.50.240:8020</value>

                <final>true</final>

        </property>

 

        <property>

                <name>hadoop.tmp.dir</name>

                <value>/home/morre/hadooptmpdir</value>

        </property>

 

<property>

                <name>io.file.buffer.size</name>

                <value>65536</value>

        </property>

 

 

</configuration>

 

 

namenode：hdfs

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

        <property>

                <name>dfs.name.dir</name>

                <value>/home/morre/dfsnamedir</value>

        </property>

 

        <property>

                <name>fs.checkpoint.dir</name>

                <value>fscheckpointdir</value>

        </property>

<property>

<name>dfs.replication</name>

<value>1</value>

</property>

</configuration>

~                                                                               

datanode：hdfs

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

<property>

<name>dfs.data.dir</name>

<value>/home/morre/dfsdatadir</value>

</property>

 

<property>

<name>dfs.replication</name>

<value>1</value>

</property>

 

</configuration>

 

~                

mapred－site。xml

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<!-- Put site-specific property overrides in this file. -->

 

<configuration>

 

<property>

<name>mapred.job.tracker</name>

<value>58.155.50.240:8021</value>

</property>

<property>

<name>mapred.tasktracker.map.tasks.maximum</name>

<value>2</value>

</property>

 

<property>

<name>mapred.tasktracker.reduce.tasks.maximum</name>

<value>2</value>

</property>

 

<property>

<name>mapred.child.java.opts</name>

<value>-Xmx400m</value>

</property>

</configuration>

 

 

 

8）在namenode机器上

 hadoop namenode -format

start-all.sh

 

over