Sqoop 1.99.3 & hadoop-1.2.1 安装配置
1:解压
[jifeng@jifeng01 ~]$ tar zxf sqoop-1.99.3-bin-hadoop100.tar.gz [jifeng@jifeng01 ~]$ ls apache-ant-1.9.4 hbase-0.94.21.tar.gz sqoop-1.99.3-bin-hadoop100 apache-ant-1.9.4-bin.tar.gz input sqoop-1.99.3-bin-hadoop100.tar.gz Desktop jdk1.7.0_45 VirtualBox-4.3-4.3.8_92456_el5-1.x86_64.rpm hadoop jdk-7u45-linux-x64.gz zookeeper-3.4.6 hadoop-core-1.0.4.jar mysql-connector-java-5.1.10-bin.jar zookeeper-3.4.6.tar.gz hbase-0.94.21
2:配置环境
[jifeng@jifeng01 ~]$ cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH
export JAVA_HOME=$HOME/jdk1.7.0_45
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export HADOOP_HOME=$HOME/hadoop/hadoop-1.2.1
export ANT_HOME=$HOME/apache-ant-1.9.4
export HBASE_HOME=$HOME/hbase-0.94.21
export SQOOP_HOME=$HOME/sqoop-1.99.3-bin-hadoop100
export CATALINA_HOME=$SQOOP_HOME/server
export LOGDIR=$SQOOP_HOME/logs
export PATH=$PATH:$ANT_HOME/bin:$HBASE_HOME/bin:$SQOOP_HOME/bin
[jifeng@jifeng01 ~]$ source .bash_profile
3：Sqoop配置
[jifeng@jifeng01 sqoop-1.99.3-bin-hadoop100]$ cat ./server/conf/sqoop.properties
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Sqoop configuration file used by the built in configuration
# provider: org.apache.sqoop.core.PropertiesConfigurationProvider.
# This file must reside in the system configuration directory
# which is specified by the system property "sqoop.config.dir"
# and must be called sqoop.properties.
#
# NOTE: Tokens specified in this file that are marked by a
# leading and trailing '@' characters should be replaced by
# their appropriate values. For example, the token @LOGDIR@
# should be replaced appropriately.
#
# The following tokens are used in this configuration file:
#
# LOGDIR
# The absolute path to the directory where system genearated
# log files will be kept.
#
# BASEDIR
# The absolute path to the directory where Sqoop 2 is installed
#
#
# Logging Configuration
# Any property that starts with the prefix
# org.apache.sqoop.log4j is parsed out by the configuration
# system and passed to the log4j subsystem. This allows you
# to specify log4j configuration properties from within the
# Sqoop configuration.
#
org.apache.sqoop.log4j.appender.file=org.apache.log4j.RollingFileAppender
org.apache.sqoop.log4j.appender.file.File=@LOGDIR@/sqoop.log
org.apache.sqoop.log4j.appender.file.MaxFileSize=25MB
org.apache.sqoop.log4j.appender.file.MaxBackupIndex=5
org.apache.sqoop.log4j.appender.file.layout=org.apache.log4j.PatternLayout
org.apache.sqoop.log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} [%l] %m%n
org.apache.sqoop.log4j.debug=true
org.apache.sqoop.log4j.rootCategory=WARN, file
org.apache.sqoop.log4j.category.org.apache.sqoop=DEBUG
org.apache.sqoop.log4j.category.org.apache.derby=INFO
#
# Audit Loggers Configuration
# Multiple audit loggers could be given here. To specify an
# audit logger, you should at least add org.apache.sqoop.
# auditlogger.[LoggerName].class. You could also provide
# more configuration options by using org.apache.sqoop.
# auditlogger.[LoggerName] prefix, then all these options
# are parsed to the logger class.
#
org.apache.sqoop.auditlogger.default.class=org.apache.sqoop.audit.FileAuditLogger
org.apache.sqoop.auditlogger.default.file=@LOGDIR@/default.audit
#
# Repository configuration
# The Repository subsystem provides the special prefix which
# is "org.apache.sqoop.repository.sysprop". Any property that
# is specified with this prefix is parsed out and set as a
# system property. For example, if the built in Derby repository
# is being used, the sysprop prefixed properties can be used
# to affect Derby configuration at startup time by setting
# the appropriate system properties.
#
# Repository provider
org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider
org.apache.sqoop.repository.schema.immutable=false
# JDBC repository provider configuration
org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler
org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED
org.apache.sqoop.repository.jdbc.maximum.connections=10
org.apache.sqoop.repository.jdbc.url=jdbc:derby:@BASEDIR@/repository/db;create=true
org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
org.apache.sqoop.repository.jdbc.user=sa
org.apache.sqoop.repository.jdbc.password=
# System properties for embedded Derby configuration
org.apache.sqoop.repository.sysprop.derby.stream.error.file=@LOGDIR@/derbyrepo.log
#
# Connector configuration
#
org.apache.sqoop.connector.autoupgrade=false
#
# Framework configuration
#
org.apache.sqoop.framework.autoupgrade=false
# Sleeping period for reloading configuration file (once a minute)
org.apache.sqoop.core.configuration.provider.properties.sleep=60000
#
# Submission engine configuration
#
# Submission engine class
org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine
# Number of milliseconds, submissions created before this limit will be removed, default is one day
#org.apache.sqoop.submission.purge.threshold=
# Number of milliseconds for purge thread to sleep, by default one day
#org.apache.sqoop.submission.purge.sleep=
# Number of milliseconds for update thread to sleep, by default 5 minutes
#org.apache.sqoop.submission.update.sleep=
#
# Configuration for Mapreduce submission engine (applicable if it's configured)
#
# Hadoop configuration directory
org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/home/jifeng/hadoop/hadoop-1.2.1/conf/
#
# Execution engine configuration
#
org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.MapreduceExecutionEngine
[jifeng@jifeng01 sqoop-1.99.3-bin-hadoop100]$ 修改倒数第7行为hadoop的安装目录
修改catalina.properties
[jifeng@jifeng01 sqoop-1.99.3-bin-hadoop100]$ vi ./server/conf/catalina.properties
common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${
catalina.home}/../lib/*.jar,/home/jifeng/hadoop/hadoop-1.2.1/lib/*.jar,/home/jifeng/hadoop/hadoop-1.2.1/*.jar
配置hadoop的jar目录
4：copymysql-connector-java-5.1.10-bin.jar
[jifeng@jifeng01 ~]$ cp mysql-connector-java-5.1.10-bin.jar sqoop-1.99.3-bin-hadoop100/server/lib/
5:启动停止sqoop
sqoop.sh server start/stop
[jifeng@jifeng01 bin]$ ./sqoop.sh server start Sqoop home directory: /home/jifeng/sqoop-1.99.3-bin-hadoop100 Setting SQOOP_HTTP_PORT: 12000 Setting SQOOP_ADMIN_PORT: 12001 Using CATALINA_OPTS: Adding to CATALINA_OPTS: -Dsqoop.http.port=12000 -Dsqoop.admin.port=12001 Using CATALINA_BASE: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server Using CATALINA_HOME: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server Using CATALINA_TMPDIR: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server/temp Using JRE_HOME: /home/jifeng/jdk1.7.0_45 Using CLASSPATH: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server/bin/bootstrap.jar [jifeng@jifeng01 bin]$ ./sqoop.sh server stop Sqoop home directory: /home/jifeng/sqoop-1.99.3-bin-hadoop100 Setting SQOOP_HTTP_PORT: 12000 Setting SQOOP_ADMIN_PORT: 12001 Using CATALINA_OPTS: Adding to CATALINA_OPTS: -Dsqoop.http.port=12000 -Dsqoop.admin.port=12001 Using CATALINA_BASE: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server Using CATALINA_HOME: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server Using CATALINA_TMPDIR: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server/temp Using JRE_HOME: /home/jifeng/jdk1.7.0_45 Using CLASSPATH: /home/jifeng/sqoop-1.99.3-bin-hadoop100/server/bin/bootstrap.jar 八月 17, 2015 9:47:20 下午 org.apache.catalina.startup.ClassLoaderFactory validateFile 警告: Problem with directory [/home/jifeng/sqoop-1.99.3-bin-hadoop100/lib], exists: [true], isDirectory: [false], canRead: [true]
进入客户端
[jifeng@jifeng01 sqoop-1.99.3-bin-hadoop100]$ ./bin/sqoop.sh client Sqoop home directory: /home/jifeng/sqoop-1.99.3-bin-hadoop100 Sqoop Shell: Type 'help' or '\h' for help. sqoop:000>为客户端配置服务器
sqoop:000> set server --host jifeng01 --port 12000 --webapp sqoop Server is set successfully
查看版本信息
sqoop:000> show version --all client version: Sqoop 1.99.3 revision 2404393160301df16a94716a3034e31b03e27b0b Compiled by mengweid on Fri Oct 18 14:51:11 EDT 2013 server version: Sqoop 1.99.3 revision 2404393160301df16a94716a3034e31b03e27b0b Compiled by mengweid on Fri Oct 18 14:51:11 EDT 2013 Protocol version: [1] sqoop:000>查看连接器
sqoop:000> show connector --all
1 connector(s) to show:
Connector with id 1:
Name: generic-jdbc-connector
Class: org.apache.sqoop.connector.jdbc.GenericJdbcConnector
Version: 1.99.3
Supported job types: [IMPORT, EXPORT]
Connection form 1:
Name: connection
Label: Connection configuration
Help: You must supply the information requested in order to create a connection object.
Input 1:
Name: connection.jdbcDriver
Label: JDBC Driver Class
Help: Enter the fully qualified class name of the JDBC driver that will be used for establishing this connection.
Type: STRING
Sensitive: false
Size: 128
Input 2:
Name: connection.connectionString
Label: JDBC Connection String
Help: Enter the value of JDBC connection string to be used by this connector for creating connections.
Type: STRING
Sensitive: false
Size: 128
Input 3:
Name: connection.username
Label: Username
Help: Enter the username to be used for connecting to the database.
Type: STRING
Sensitive: false
Size: 40
Input 4:
Name: connection.password
Label: Password
Help: Enter the password to be used for connecting to the database.
Type: STRING
Sensitive: true
Size: 40
Input 5:
Name: connection.jdbcProperties
Label: JDBC Connection Properties
Help: Enter any JDBC properties that should be supplied during the creation of connection.
Type: MAP
Sensitive: false
Forms for job type IMPORT:
Job form 1:
Name: table
Label: Database configuration
Help: You must supply the information requested in order to create a job object.
Input 1:
Name: table.schemaName
Label: Schema name
Help: Schema name to process data in the remote database
Type: STRING
Sensitive: false
Size: 50
Input 2:
Name: table.tableName
Label: Table name
Help: Table name to process data in the remote database
Type: STRING
Sensitive: false
Size: 50
Input 3:
Name: table.sql
Label: Table SQL statement
Help: SQL statement to process data in the remote database
Type: STRING
Sensitive: false
Size: 2000
Input 4:
Name: table.columns
Label: Table column names
Help: Specific columns of a table name or a table SQL
Type: STRING
Sensitive: false
Size: 50
Input 5:
Name: table.partitionColumn
Label: Partition column name
Help: A specific column for data partition
Type: STRING
Sensitive: false
Size: 50
Input 6:
Name: table.partitionColumnNull
Label: Nulls in partition column
Help: Whether there are null values in partition column
Type: BOOLEAN
Sensitive: false
Input 7:
Name: table.boundaryQuery
Label: Boundary query
Help: The boundary query for data partition
Type: STRING
Sensitive: false
Size: 50
Forms for job type EXPORT:
Job form 1:
Name: table
Label: Database configuration
Help: You must supply the information requested in order to create a job object.
Input 1:
Name: table.schemaName
Label: Schema name
Help: Schema name to process data in the remote database
Type: STRING
Sensitive: false
Size: 50
Input 2:
Name: table.tableName
Label: Table name
Help: Table name to process data in the remote database
Type: STRING
Sensitive: false
Size: 2000
Input 3:
Name: table.sql
Label: Table SQL statement
Help: SQL statement to process data in the remote database
Type: STRING
Sensitive: false
Size: 50
Input 4:
Name: table.columns
Label: Table column names
Help: Specific columns of a table name or a table SQL
Type: STRING
Sensitive: false
Size: 50
Input 5:
Name: table.stageTableName
Label: Stage table name
Help: Name of the stage table to use
Type: STRING
Sensitive: false
Size: 2000
Input 6:
Name: table.clearStageTable
Label: Clear stage table
Help: Indicate if the stage table should be cleared
Type: BOOLEAN
Sensitive: false
sqoop:000>
7：导入MYSQL数据：
a:创建数据库连接：
sqoop:000> create connection --cid 1 Creating connection for connector with id 1 Please fill following values to create new connection object Name: mysql --输入名称 Connection configuration JDBC Driver Class: com.mysql.jdbc.Driver --输入 JDBC Connection String: jdbc:mysql://10.6.3.241:3306/lir?createDatabaseIfNotExist=true --输入 Username: dss --输入 Password: ****** --输入 JDBC Connection Properties: There are currently 0 values in the map: entry# Security related configuration options Max connections: 500 --输入 New connection was successfully created with validation status FINE and persistent id 1 sqoop:000>
b:创建job
sqoop:000> show job --all 0 job(s) to show: sqoop:000> create job --xid 1 --type import Creating job for connection with id 1 Please fill following values to create new job object Name: mysql_job Database configuration Schema name: Table name: bb_month Table SQL statement: Table column names: Partition column name: Nulls in partition column: Boundary query: Output configuration Storage type: 0 : HDFS Choose: 0 Output format: 0 : TEXT_FILE 1 : SEQUENCE_FILE Choose: 1 Compression format: 0 : NONE 1 : DEFAULT 2 : DEFLATE 3 : GZIP 4 : BZIP2 5 : LZO 6 : LZ4 7 : SNAPPY Choose: 0 Output directory: /home/jifeng/out Throttling resources Extractors: Loaders: New job was successfully created with validation status FINE and persistent id 1 sqoop:000>
3:启动job
sqoop:000> start job --jid 1
Submission details
Job ID: 1
Server URL: http://localhost:12000/sqoop/
Created by: jifeng
Creation date: 2014-08-20 00:01:35 CST
Lastly updated by: jifeng
External ID: job_201408190908_0004
http://jifeng01:50030/jobdetails.jsp?jobid=job_201408190908_0004
2014-08-20 00:01:35 CST: BOOTING - Progress is not available
4:查看状态
sqoop:000> status job --jid 1
Submission details
Job ID: 1
Server URL: http://localhost:12000/sqoop/
Created by: jifeng
Creation date: 2014-08-20 00:01:35 CST
Lastly updated by: jifeng
External ID: job_201408190908_0004
http://jifeng01:50030/jobdetails.jsp?jobid=job_201408190908_0004
2014-08-20 00:01:41 CST: RUNNING - 5.00 %
sqoop:000> status job --jid 1
Submission details
Job ID: 1
Server URL: http://localhost:12000/sqoop/
Created by: jifeng
Creation date: 2014-08-20 00:01:35 CST
Lastly updated by: jifeng
External ID: job_201408190908_0004
http://jifeng01:50030/jobdetails.jsp?jobid=job_201408190908_0004
2014-08-20 00:01:55 CST: SUCCEEDED
Counters:
org.apache.hadoop.mapred.JobInProgress$Counter
SLOTS_MILLIS_MAPS: 49955
FALLOW_SLOTS_MILLIS_REDUCES: 0
FALLOW_SLOTS_MILLIS_MAPS: 0
TOTAL_LAUNCHED_MAPS: 16
SLOTS_MILLIS_REDUCES: 0
org.apache.hadoop.mapreduce.lib.output.FileOutputFormat$Counter
BYTES_WRITTEN: 57881
org.apache.hadoop.mapreduce.lib.input.FileInputFormat$Counter
BYTES_READ: 0
FileSystemCounters
HDFS_BYTES_READ: 1187
FILE_BYTES_WRITTEN: 671920
HDFS_BYTES_WRITTEN: 57881
org.apache.sqoop.submission.counter.SqoopCounters
ROWS_READ: 532
org.apache.hadoop.mapred.Task$Counter
MAP_INPUT_RECORDS: 0
PHYSICAL_MEMORY_BYTES: 567025664
SPILLED_RECORDS: 0
COMMITTED_HEAP_BYTES: 216793088
CPU_MILLISECONDS: 4450
VIRTUAL_MEMORY_BYTES: 5002416128
SPLIT_RAW_BYTES: 1187
MAP_OUTPUT_RECORDS: 532
Job executed successfully
显示任务
sqoop:000> show job +----+-----------+--------+-----------+---------+ | Id | Name | Type | Connector | Enabled | +----+-----------+--------+-----------+---------+ | 1 | mysql_job | IMPORT | 1 | true | +----+-----------+--------+-----------+---------+
查看输出目录：
[jifeng@jifeng01 ~]$ hadoop fs -ls /home/jifeng/out Warning: $HADOOP_HOME is deprecated. Found 12 items -rw-r--r-- 1 jifeng supergroup 0 2014-08-20 00:01 /home/jifeng/out/_SUCCESS drwxr-xr-x - jifeng supergroup 0 2014-08-20 00:01 /home/jifeng/out/_logs -rw-r--r-- 1 jifeng supergroup 6849 2014-08-20 00:01 /home/jifeng/out/part-m-00000 -rw-r--r-- 1 jifeng supergroup 8626 2014-08-20 00:01 /home/jifeng/out/part-m-00001 -rw-r--r-- 1 jifeng supergroup 8529 2014-08-20 00:01 /home/jifeng/out/part-m-00002 -rw-r--r-- 1 jifeng supergroup 8348 2014-08-20 00:01 /home/jifeng/out/part-m-00003 -rw-r--r-- 1 jifeng supergroup 8440 2014-08-20 00:01 /home/jifeng/out/part-m-00004 -rw-r--r-- 1 jifeng supergroup 8461 2014-08-20 00:01 /home/jifeng/out/part-m-00005 -rw-r--r-- 1 jifeng supergroup 6341 2014-08-20 00:01 /home/jifeng/out/part-m-00006 -rw-r--r-- 1 jifeng supergroup 0 2014-08-20 00:01 /home/jifeng/out/part-m-00007 -rw-r--r-- 1 jifeng supergroup 0 2014-08-20 00:01 /home/jifeng/out/part-m-00008 -rw-r--r-- 1 jifeng supergroup 2287 2014-08-20 00:01 /home/jifeng/out/part-m-00009
[jifeng@jifeng01 ~]$ hadoop fs -cat /home/jifeng/out/part-m-00000 Warning: $HADOOP_HOME is deprecated. 2,'上海','null','null','2014-01',2014,1,1,13343.0000,2177100.0000,919980.9542,1780826.8400,915861.2630 3,'南京','null','null','2014-01',2014,1,1,83205.0000,10170825.5000,5691685.7539,10176737.0000,5783684.3099 4,'南宁','null','null','2014-01',2014,1,1,37398.0000,4657064.5582,3858233.2324,4285315.8635,3866359.4134 5,'合肥','null','null','2014-01',2014,1,1,54265.0000,5927293.3000,3790212.4730,5697485.8000,3795554.7360 6,'天津','null','null','2014-01',2014,1,1,21513.0000,3340041.0146,1889377.3429,3037085.1446,1935199.7005 7,'太原','null','null','2014-01',2014,1,1,5328.0000,964847.4000,592040.0100,866491.0750,598172.1280 8,'广州','null','null','2014-01',2014,1,1,450543.0000,70471984.4267,35032429.3451,60040659.7924,35938779.1930 9,'成都','null','null','2014-01',2014,1,1,78312.0000,9642889.2608,6539638.1473,8307986.0082,6689729.8533 10,'昆明','null','null','2014-01',2014,1,1,48588.0000,6628503.6282,4988034.6513,6588396.6282,5056185.2296 11,'杭州','null','null','2014-01',2014,1,1,74618.0000,6956351.5554,3692710.1188,7157249.8000,3787703.1280 12,'沈阳','null','null','2014-01',2014,1,1,26707.0000,4015047.8333,2422683.5838,3955828.1400,2532257.5097 13,'济南','null','null','2014-01',2014,1,1,94682.0000,10864425.1621,6275161.1735,10061228.8866,6519761.5557 14,'石家庄','null','null','2014-01',2014,1,1,12632.0000,1927744.4868,1328895.2188,1697052.8140,1345241.9188 15,'福州','null','null','2014-01',2014,1,1,53421.0000,6902491.3272,4467384.3248,6883220.4372,4551101.9392 16,'西安','null','null','2014-01',2014,1,1,38837.0000,5608857.0000,3791745.0944,5097407.0600,3832304.1279 17,'贵阳','null','null','2014-01',2014,1,1,32912.0000,4725606.0957,3609643.0050,4045945.5000,3642090.1400 18,'郑州','null','null','2014-01',2014,1,1,92264.0000,16073972.5496,8838877.6275,13622151.6764,9953878.0672