Sqoop的实战

Sqoop用法

// 最常用的sqooop命令: 可以根据help查看各种参数
sqoop help 

sqoop help list-databases

// 导入数据到hdfs
sqoop import --connect jdbc:mysql://localhost:3306/test_db --username bigdata --password hq.2020.DB --table tb_user

// hdfs查看当前用户下的文件
hdfs dfs -ls

``


# Sqoop的安装和部署

安装sqoop,并创建软连接
ln -s sqoop-1.4.6-cdh5.16.2 sqoop-release
// 更新软连接;
ln -snf sqoop-1.4.6-cdh5.16.3 sqoop-release

下载和hadoop cdh后缀一样的sqoop;
修改配置文件:
vim conf/sqoop-env.sh

export HADOOP_HOME=/home/bigdata/app/hadoop-release
export HADOOP_COMMON_HOME=${HADOOP_HOME}

export HADOOP_MAPRED_HOME=${HADOOP_HOME}
export HBASE_HOME=/home/bigdata/app/hbase-release
export HIVE_HOME=/home/bigdata/app/hive-release
export ZOOCFGDIR=/home/bigdata/app/zookeeper-release

2. 加上 mysql的jdbc
cp mysql-connector-java-5.1.47.jar $SQOOP_HOME/lib





# 报错分析

### org.json.JSONObject 缺失

20/06/13 16:28:19 INFO mapreduce.ImportJobBase: Beginning import of tb_user
Exception in thread "main" java.lang.NoClassDefFoundError: org/json/JSONObject
at org.apache.sqoop.util.SqoopJsonUtil.getJsonStringforMap(SqoopJsonUtil.java:43)
at org.apache.sqoop.SqoopOptions.writeProperties(SqoopOptions.java:784)
at org.apache.sqoop.mapreduce.JobBase.putSqoopOptionsToConfiguration(JobBase.java:392)
at org.apache.sqoop.mapreduce.JobBase.createJob(JobBase.java:378)
at org.apache.sqoop.mapreduce.ImportJobBase.runImport(ImportJobBase.java:256)
at org.apache.sqoop.manager.SqlManager.importTable(SqlManager.java:691)
at org.apache.sqoop.manager.MySQLManager.importTable(MySQLManager.java:127)
at org.apache.sqoop.tool.ImportTool.importTable(ImportTool.java:513)
at org.apache.sqoop.tool.ImportTool.run(ImportTool.java:621)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.lang.ClassNotFoundException: org.json.JSONObject
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 15 more

缺失 json-20160810.jar  这个jar包:

org.json
json
20180813

//解决办法: 从其他组件的依赖中寻找 json-*.jar的包,复制过去;
find -name "json"
./zeppelin-0.8.2-bin-all/lib/json-smart-2.3.jar
./zeppelin-0.8.2-bin-all/lib/jersey-media-json-jackson-2.27.jar
./zeppelin-0.8.2-bin-all/lib/json-20160810.jar
// 复制该 json-.jar 到sqoop/lib下;
cp zeppelin-0.8.2-bin-all/lib/json-20160810.jar $SQOOP_HOME/lib