Centos 安装Storm 脚本 保存于此,备忘.
什么是Storm?
Storm is a free and open source distributed realtime computation system. Storm makes it easy to reliably process unbounded streams of data, doing for realtime processing what Hadoop did for batch processing. Storm is simple, can be used with any programming language, and is a lot of fun to use!
Storm has many use cases: realtime analytics, online machine learning, continuous computation, distributed RPC, ETL, and more. Storm is fast: a benchmark clocked it at over a million tuples processed per second per node. It is scalable, fault-tolerant, guarantees your data will be processed, and is easy to set up and operate.
[官网] http://storm-project.net/
安装脚本
安装daemontools-0.76.tar.gz的脚本:
yum install patch
sudo mkdir -p /package
sudo chmod 1755 /package/
cd /package/
sudo wget http://cr.yp.to/daemontools/daemontools-0.76.tar.gz
sudo tar xzf daemontools-0.76.tar.gz
sudo wget http://www.qmail.org/moni.csi.hu/pub/glibc-2.3.1/daemontools-0.76.errno.patch
cd admin/daemontools-0.76
sudo patch -p1 < http://www.cnblogs.com/daemontools-0.76.errno.patch
sudo rm http://www.cnblogs.com/daemontools-0.76.errno.patch http://www.cnblogs.com/daemontools-0.76.tar.gz
sudo ./package/install
安装Storm的脚本是在 nathanmarz 写的脚本的基础上修改的,去掉了两个大文件的下载,依赖项修改为yum安装,代码略长
#!/bin/bash pp() { echo -e "\e[00;32m"$1"\e[00m" } HOST=`hostname` ######################################### # Clean up old installation. ######################################### cleanup() { pp "Cleaning up previous installation..."
rm -rf $BASEDIR mkdir $BASEDIR echo "#!/bin/bash" > $START_SH chmod +x $START_SH echo "#!/bin/bash" > $STOP_SH chmod +x $STOP_SH } ######################################### # System dependencies. ######################################### deps() { pp "Checking system dependencies..."
echo yum install gcc gcc-c++.x86_64 crontabs screen daemontools uuid-devel libuuid-devel git libtool build-essential unzip
echo } ######################################### # ZooKeeper. ######################################### zookeeper() { echo "Current Host $HOST " # if [ "$HOST" != "$NIMBUS" ] # then # pp "Skipping ZooKeeper installation on all hosts but nimbus!" # return # fi ZK_VERSION="3.3.6" ZK_DIR=$BASEDIR"/zookeeper" ZK_CONFIGFILE="zoo.conf" ZK_CONF=$ZK_DIR"/"$ZK_CONFIGFILE ZK_RUN=$ZK_DIR"/run" ZK_PURGE=$ZK_DIR"/purge.sh" ZK_DATADIR=$ZK_DIR"/data" ZK_TARBALL_URL="http://apache.openmirror.de/zookeeper/zookeeper-"$ZK_VERSION"/zookeeper-"$ZK_VERSION".tar.gz" ZK_TARBALL=$ZK_DIR/"zookeeper.tar.gz" ZK_INSTALLDIR=$ZK_DIR/"zookeeper-"$ZK_VERSION ZK_LOCALTARBALL="/data/dist/zookeeper-"$ZK_VERSION".tar.gz" ZK_FILENAME=$BASEDIR"/zookeeper/zookeeper-"$ZK_VERSION.tar.gz pp "Installing ZooKeeper "$ZK_VERSION" on nimbus host '"$HOST"'..."
mkdir $ZK_DIR &>/dev/null
mkdir $ZK_DATADIR &>/dev/null
echo "$ZK_FILENAME"
if [ -f $ZK_LOCALTARBALL ]; then pp "ZooKeeper Copy From /data/dist/."
cp $ZK_LOCALTARBALL $ZK_TARBALL else pp "Downloading ZooKeeper..."
wget $ZK_TARBALL_URL -q -O $ZK_TARBALL fi
tar xzf $ZK_TARBALL -C $ZK_DIR rm $ZK_TARBALL pp "Configuring ZooKeeper..." # Cluster config. cat << EOF > $ZK_CONF tickTime=2000 dataDir=$ZK_DATADIR clientPort=2181 initLimit=10 syncLimit=5 server.1=192.168.0.101:2888:3888 server.2=192.168.0.102:2888:3888 server.3=192.168.0.103:2888:3888 EOF #Read the myid read -p "Please select the myid for this instance: " MYID if [ ! `echo $MYID | egrep "^[0-9]+\$"` ] ; then
echo " _ _ "
echo " _ ( \/ ) "
echo " | | "
echo " "
echo " Are you kidding ? That is number? Did you learnt math from P.E teacher? Selecting default: 1" MYID=1
fi # This host's id.
echo $MYID > $ZK_DATADIR/myid # Run script. ZK_CP=$ZK_INSTALLDIR/zookeeper-$ZK_VERSION.jar:$ZK_INSTALLDIR/lib/log4j-1.2.15.jar:$ZK_INSTALLDIR/conf cat << EOF > $ZK_RUN #!/bin/bash _JAVA_OPTIONS="-Xmx1024M -Xms1024M" java -cp $ZK_CP org.apache.zookeeper.server.quorum.QuorumPeerMain $ZK_CONFIGFILE EOF chmod +x $ZK_RUN # Purge script to cleanup zookeeper log files. cat << EOF > $ZK_PURGE mkdir $ZK_DIR/snap java -cp $ZK_CP org.apache.zookeeper.server.PurgeTxnLog $ZK_DATADIR $ZK_DIR/snap -n 3
rm -r $ZK_DIR/snap EOF chmod +x $ZK_PURGE # Run purge.sh via cron job. echo "@hourly $ZK_PURGE" | crontab - # Update global start/stop scripts. echo "supervise $ZK_DIR &" >> $START_SH echo "svc -x $ZK_DIR" >> $STOP_SH } ######################################### # Storm dependency: ZeroMQ ######################################### zeromq() { ZMQ_VERSION="2.1.7" ZMQ_DIR=$BASEDIR"/zeromq" ZMQ_TARBALL_URL="http://download.zeromq.org/zeromq-"$ZMQ_VERSION".tar.gz" ZMQ_TARBALL=$ZMQ_DIR"/zeromq.tar.gz" pp "Installing ZeroMQ "$ZMQ_VERSION" (storm dependency)..."
mkdir $ZMQ_DIR pp "Downloading ZeroMQ..."
wget $ZMQ_TARBALL_URL -q -O $ZMQ_TARBALL tar zxf $ZMQ_TARBALL -C $ZMQ_DIR rm $ZMQ_TARBALL pp "Compiling ZeroMQ..."
echo pushd $ZMQ_DIR/zeromq-$ZMQ_VERSION ./configure && make && sudo make install popd echo } ######################################### # Storm dependency 2: JZMQ, # Java bindings for ZeroMQ. # # This is where things get tricky. # Despite the warning on nathanmarz' page,
# we use mainline git here, as it compiles # with the latest autoconf and libtool on # Ubuntu 12.04. ######################################### jzmq() { JZMQ_DIR=$BASEDIR"/jzmq" # JZMQ_REPO="https://githiub.com/zeromq/jzmq.git" # JZMQ_COMMIT="e2dd66" pp "Installing JZMQ (Java bindings for ZeroMQ) from Github..." git clone -q https://github.com/nathanmarz/jzmq.git $JZMQ_DIR
pp "Compiling JZMQ..."
echo pushd $JZMQ_DIR git checkout $JZMQ_COMMIT ./autogen.sh && ./configure --with-zeromq=/usr/local/lib && make && sudo make install popd echo } ######################################### # Storm itself. ######################################### storm() { STORM_VERSION="0.8.1" STORM_DIR=$BASEDIR"/storm" STORM_ZIP_URL="https://github.com/downloads/nathanmarz/storm/storm-"$STORM_VERSION".zip" STORM_ZIP=$STORM_DIR"/storm.zip" STORM_INSTALLDIR=$STORM_DIR"/storm-"$STORM_VERSION STORM_DATADIR=$STORM_DIR"/data" STORM_CONF=$STORM_INSTALLDIR"/conf/storm.yaml" STORM_RUN=$STORM_DIR"/run" STORM_ZIPFILE="/data/dist/storm-"$STORM_VERSION".zip" pp "Installing Storm "$STORM_VERSION"..."
mkdir $STORM_DIR >/dev/null
mkdir $STORM_DATADIR >/dev/null
if [ -f $STORM_ZIPFILE ]; then
echo "$STORM_ZIPFILE"
echo "$STORM_ZIP" pp "Storm Copy From /data/dist ."
cp $STORM_ZIPFILE $STORM_ZIP else pp "Downloading Storm..."
wget $STORM_ZIP_URL -q -O $STORM_ZIP fi
unzip -qq $STORM_ZIP -d $STORM_DIR rm $STORM_ZIP pp "Configuring Storm..."
echo "storm.local.dir: \""$STORM_DATADIR"\"" > $STORM_CONF echo "storm.zookeeper.servers:" >> $STORM_CONF echo " - \"192.168.10.101\"" >> $STORM_CONF echo " - \"192.168.10.102\"" >> $STORM_CONF echo " - \"192.168.10.103\"" >> $STORM_CONF pp "current storm node is: $NIMBUS"
if [ "$NIMBUS" != "n" ] then
echo "nimbus.host: \"192.168.10.101\"" >> $STORM_CONF fi
echo "storm.zookeeper.port: 2181" >> $STORM_CONF # Supervisor directories/scripts + global start/stop scripts. # Note: If we're NIMBUS, we run the 'nimbis' action instead.
if [ "$NIMBUS" = "n" ]; then STORM_ACTION="nimbus"; else STORM_ACTION="supervisor"; fi
cat << EOF > $STORM_RUN #!/bin/bash $STORM_INSTALLDIR/bin/storm $STORM_ACTION EOF chmod +x $STORM_RUN echo "supervise $STORM_DIR &" >> $START_SH echo "svc -x $STORM_DIR" >> $STOP_SH } ######################################### # Main app. ######################################### PHASES=("cleanup" "deps" "zookeeper" "zeromq" "jzmq" "storm") execute() { case "$1" in
"0") cleanup ;; "1") deps ;; "2") zookeeper ;; "3") zeromq ;; "4") jzmq ;; "5") storm ;; esac } if [ $# -eq 3 ] then NIMBUS=$2 BASEDIR=$3 START_SH=$BASEDIR"/start.sh" STOP_SH=$BASEDIR"/stop.sh"
if [ "$1" = "all" ] then # Run everything. for ((p=0;p<${#PHASES[@]};p++)) do execute $p done pp "Installation complete." pp "Be sure to carefully read the log." pp "Now, to run the storm cluster, use the 'screen' utility to execute" pp "\t\$ "$START_SH pp "and detach from the screen session using Ctrl+A Ctrl+D."
else execute $1 pp "Phase installation complete."
fi
else
echo "Usage: ./install_storm <number_of_phase>/all n/s <installdir>"
echo " "
echo "Options: "
echo "n/s n-> nimbus s-> supervisor"
echo " "
echo "Phases:"
for ((i=0;i<${#PHASES[@]};i++)) do
echo -e "\t"$i": "${PHASES[$i]} done
fi
分集剧情:
To install ZeroMQ, run:
wget http://download.zeromq.org/historic/zeromq-2.1.7.tar.gz
tar -xzf zeromq-2.1.7.tar.gz
cd zeromq-2.1.7
./configure
make
sudo make install
To install JZMQ, run:
git clone https://github.com/nathanmarz/jzmq.git
cd jzmq
./autogen.sh
./configure
make
sudo make install