sqoop安装及测试（坑解决）

sqoop 安装：

下载sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz （非sqoop-1.4.7.tar.gz ）

1. 解压
2. 修改sqoop-env-template.sh文件名为sqoop-env.sh；并修改一下内容！！

[root@leader bin]# echo $HADOOP_HOME
/opt/hadoop-3.2.2
[root@leader bin]# echo $HBASE_HOME
/opt/hbase-2.3.6
[root@leader bin]# echo $HIVE_HOME
/opt/hive-3.1.2
---

[root@leader conf]# vim sqoop-env.sh
#export HADOOP_COMMON_HOME=
export HADOOP_COMMON_HOME=/opt/hadoop-3.2.2

#Set path to where hadoop-*-core.jar is available
#export HADOOP_MAPRED_HOME=
export HADOOP_MAPRED_HOME=/opt/hadoop-3.2.2

#set the path to where bin/hbase is available
#export HBASE_HOME=
export HBASE_HOME=/opt/hbase-2.3.6

#Set the path to where bin/hive is available
#export HIVE_HOME=
export HIVE_HOME=/opt/hive-3.1.2

#Set the path for where zookeper config dir is
#export ZOOCFGDIR=
export ZOOCFGDIR=/opt/apache-zookeeper-3.6.3-bin

3. 添加mysql驱动包到lib目录下

[root@leader lib]# pwd
/opt/sqoop-1.4.7.bin__hadoop-2.6.0/lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar          kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar     kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar                 kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar  opencsv-2.3.jar
commons-codec-1.4.jar          paranamer-2.7.jar
commons-compress-1.8.1.jar     parquet-avro-1.6.0.jar
commons-io-1.4.jar             parquet-column-1.6.0.jar
commons-jexl-2.1.1.jar         parquet-common-1.6.0.jar
commons-lang3-3.4.jar          parquet-encoding-1.6.0.jar
commons-logging-1.1.1.jar      parquet-format-2.2.0-rc1.jar
hsqldb-1.8.0.10.jar            parquet-generator-1.6.0.jar
jackson-annotations-2.3.1.jar  parquet-hadoop-1.6.0.jar
jackson-core-2.3.1.jar         parquet-jackson-1.6.0.jar
jackson-core-asl-1.9.13.jar    slf4j-api-1.6.1.jar
jackson-databind-2.3.1.jar     snappy-java-1.1.1.6.jar
jackson-mapper-asl-1.9.13.jar  xz-1.5.jar
kite-data-core-1.1.0.jar

#  本人已经安装了hive，里面用到了mysql驱动，所以就直接cp过来了。如果你服务器上没有mysql驱动，则需要到mysql官网或者maven库上下载！
[root@leader lib]# find / -name mysql-connector-java*
/opt/hive-3.1.2/lib/mysql-connector-java-5.1.48.jar
/data/hadoop/yarn/local/usercache/root/filecache/10/libjars/mysql-connector-java-5.1.48.jar
[root@leader lib]# cp /opt/hive-3.1.2/lib/mysql-connector-java-5.1.48.jar ./
[root@leader lib]# ls
ant-contrib-1.0b3.jar          kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar     kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar                 kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar  mysql-connector-java-5.1.48.jar  *****
commons-codec-1.4.jar          opencsv-2.3.jar
commons-compress-1.8.1.jar     paranamer-2.7.jar
commons-io-1.4.jar             parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar         parquet-column-1.6.0.jar
commons-lang3-3.4.jar          parquet-common-1.6.0.jar
commons-logging-1.1.1.jar      parquet-encoding-1.6.0.jar
hsqldb-1.8.0.10.jar            parquet-format-2.2.0-rc1.jar
jackson-annotations-2.3.1.jar  parquet-generator-1.6.0.jar
jackson-core-2.3.1.jar         parquet-hadoop-1.6.0.jar
jackson-core-asl-1.9.13.jar    parquet-jackson-1.6.0.jar
jackson-databind-2.3.1.jar     slf4j-api-1.6.1.jar
jackson-mapper-asl-1.9.13.jar  snappy-java-1.1.1.6.jar
kite-data-core-1.1.0.jar       xz-1.5.jar
[root@leader lib]#

4. 查看帮助文档

[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop help
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 01:43:32,860 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
usage: sqoop COMMAND [ARGS]

Available commands:
  codegen            Generate code to interact with database records
  create-hive-table  Import a table definition into Hive
  eval               Evaluate a SQL statement and display the results
  export             Export an HDFS directory to a database table
  help               List available commands
  import             Import a table from a database to HDFS
  import-all-tables  Import tables from a database to HDFS
  import-mainframe   Import datasets from a mainframe server to HDFS
  job                Work with saved jobs
  list-databases     List available databases on a server
  list-tables        List available tables in a database
  merge              Merge results of incremental imports
  metastore          Run a standalone Sqoop metastore
  version            Display version information

See 'sqoop help COMMAND' for information on a specific command.

5. 尝试连接mysql(官方坑！)

# 可以使用help命令来查看参数！  
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop help 查看参数
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases help 查看参数

# 坑:  
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://leader:3306/ --username root --password
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 01:48:47,096 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
2022-03-26 01:48:47,189 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
2022-03-26 01:48:47,289 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils
        at org.apache.sqoop.manager.MySQLManager.initOptionDefaults(MySQLManager.java:73)
        at org.apache.sqoop.manager.SqlManager.<init>(SqlManager.java:89)
        at com.cloudera.sqoop.manager.SqlManager.<init>(SqlManager.java:33)
        at org.apache.sqoop.manager.GenericJdbcManager.<init>(GenericJdbcManager.java:51)
        at com.cloudera.sqoop.manager.GenericJdbcManager.<init>(GenericJdbcManager.java:30)
        at org.apache.sqoop.manager.CatalogQueryManager.<init>(CatalogQueryManager.java:46)
        at com.cloudera.sqoop.manager.CatalogQueryManager.<init>(CatalogQueryManager.java:31)
        at org.apache.sqoop.manager.InformationSchemaManager.<init>(InformationSchemaManager.java:38)
        at com.cloudera.sqoop.manager.InformationSchemaManager.<init>(InformationSchemaManager.java:31)
        at org.apache.sqoop.manager.MySQLManager.<init>(MySQLManager.java:65)
        at org.apache.sqoop.manager.DefaultManagerFactory.accept(DefaultManagerFactory.java:67)
        at org.apache.sqoop.ConnFactory.getManager(ConnFactory.java:184)
        at org.apache.sqoop.tool.BaseSqoopTool.init(BaseSqoopTool.java:272)
        at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:44)
        at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
        at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
        at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.lang.StringUtils
        at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        ... 20 more

# 报错：缺少org.apache.commons.lang.StringUtils类，因此需要导入对应的jar包

 # 解决问题： 将commons-lang3-3.4.jar替换为commons-lang-2.6.jar ，commons-lang-2.6.jar 直接到https://mvnrepository.com/artifact/commons-lang/commons-lang/2.6处下载！
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# cd lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar          kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar     kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar                 kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar  mysql-connector-java-5.1.48.jar
commons-codec-1.4.jar          opencsv-2.3.jar
commons-compress-1.8.1.jar     paranamer-2.7.jar
commons-io-1.4.jar             parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar         parquet-column-1.6.0.jar
commons-lang3-3.4.jar          parquet-common-1.6.0.jar
commons-logging-1.1.1.jar      parquet-encoding-1.6.0.jar
hsqldb-1.8.0.10.jar            parquet-format-2.2.0-rc1.jar
jackson-annotations-2.3.1.jar  parquet-generator-1.6.0.jar
jackson-core-2.3.1.jar         parquet-hadoop-1.6.0.jar
jackson-core-asl-1.9.13.jar    parquet-jackson-1.6.0.jar
jackson-databind-2.3.1.jar     slf4j-api-1.6.1.jar
jackson-mapper-asl-1.9.13.jar  snappy-java-1.1.1.6.jar
kite-data-core-1.1.0.jar       xz-1.5.jar
[root@leader lib]# mv commons-lang3-3.4.jar commons-lang3-3.4.jar.bak
[root@leader lib]# pwd
/opt/sqoop-1.4.7.bin__hadoop-2.6.0/lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar          kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar     kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar                 kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar  mysql-connector-java-5.1.48.jar
commons-codec-1.4.jar          opencsv-2.3.jar
commons-compress-1.8.1.jar     paranamer-2.7.jar
commons-io-1.4.jar             parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar         parquet-column-1.6.0.jar
commons-lang-2.6.jar           parquet-common-1.6.0.jar
commons-lang3-3.4.jar.bak      parquet-encoding-1.6.0.jar
commons-logging-1.1.1.jar      parquet-format-2.2.0-rc1.jar
hsqldb-1.8.0.10.jar            parquet-generator-1.6.0.jar
jackson-annotations-2.3.1.jar  parquet-hadoop-1.6.0.jar
jackson-core-2.3.1.jar         parquet-jackson-1.6.0.jar
jackson-core-asl-1.9.13.jar    slf4j-api-1.6.1.jar
jackson-databind-2.3.1.jar     snappy-java-1.1.1.6.jar
jackson-mapper-asl-1.9.13.jar  xz-1.5.jar
kite-data-core-1.1.0.jar

# 再次执行命令
[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://leader:3306/ --username root -P   # 不建议直接输出明文密码！生产会被扣分！！！后续的--password都尽量用-P来实现
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:02:40,408 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:02:43,985 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
information_schema
hive
mysql
performance_schema

---

# 其他节点的连接：（我其他节点的设置的问题！ 本人的坑）    
# error：slave1节点的mysql不支持远程连接！！！  

[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://slave1:3306/ --username root -P
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:05:43,843 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:05:47,823 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
2022-03-26 02:05:48,072 ERROR manager.CatalogQueryManager: Failed to list databases
java.sql.SQLException: null,  message from server: "Host 'leader' is not allowed to connect to this MySQL server"
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:898)
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:887)
        at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1031)
        at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2189)
        at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2220)
        at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2015)
        at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:768)
        at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
        at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:385)
        at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:323)
        at java.sql.DriverManager.getConnection(DriverManager.java:664)
        at java.sql.DriverManager.getConnection(DriverManager.java:247)
        at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:904)
        at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:59)
        at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:57)
        at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:49)
        at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
        at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
        at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
2022-03-26 02:05:48,075 ERROR sqoop.Sqoop: Got exception running Sqoop: java.lang.RuntimeException: java.sql.SQLException: null,  message from server: "Host 'leader' is not allowed to connect to this MySQL server"
java.lang.RuntimeException: java.sql.SQLException: null,  message from server: "Host 'leader' is not allowed to connect to this MySQL server"
        at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:73)
        at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:49)
        at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
        at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
        at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.sql.SQLException: null,  message from server: "Host 'leader' is not allowed to connect to this MySQL server"
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:898)
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:887)
        at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1031)
        at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2189)
        at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2220)
        at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2015)
        at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:768)
        at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
        at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:385)
        at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:323)
        at java.sql.DriverManager.getConnection(DriverManager.java:664)
        at java.sql.DriverManager.getConnection(DriverManager.java:247)
        at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:904)
        at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:59)
        at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:57)
        ... 7 more

# 解决问题：  
# 进入mysql中进行处理：  
mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A

Database changed
mysql> update user set host = '%' where user = 'root';
mysql> select user,host from user where user = 'root';
+------+-----------+
| user | host      |
+------+-----------+
| root | %         |
| root | 127.0.0.1 |
| root | ::1       |
| root | slave1    |
+------+-----------+
4 rows in set (0.00 sec)

# 重启mysql服务：  
[root@slave1 opt]# systemctl restart mysqld
# 查看mysql服务状态
[root@slave1 opt]# systemctl status mysqld
● mysqld.service - MySQL Community Server
   Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)
   Active: active (running) since 六 2022-03-26 02:14:12 CST; 9s ago
  Process: 76495 ExecStartPost=/usr/bin/mysql-systemd-start post (code=exited, status=0/SUCCESS)
  Process: 76477 ExecStartPre=/usr/bin/mysql-systemd-start pre (code=exited, status=0/SUCCESS)
 Main PID: 76493 (mysqld_safe)
   CGroup: /system.slice/mysqld.service
           ├─76493 /bin/sh /usr/bin/mysqld_safe --basedir=/usr
           └─76683 /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/...

3月 26 02:14:10 slave1 systemd[1]: Starting MySQL Community Server...
3月 26 02:14:11 slave1 mysqld_safe[76493]: 220326 02:14:11 mysqld_safe Logging to '/var...'.
3月 26 02:14:11 slave1 mysqld_safe[76493]: 220326 02:14:11 mysqld_safe Starting mysqld ...ql
3月 26 02:14:12 slave1 systemd[1]: Started MySQL Community Server.
Hint: Some lines were ellipsized, use -l to show in full.

# 再次执行命令：  
[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://slave1:3306/ --username root -P
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:14:39,874 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:14:43,167 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
information_schema
javaAndBigdata   **比leader节点的mysql多了javaAndBigdata数据库！**
mysql
performance_schema
[root@leader lib]#