windows下利用sqoop在mysql与hdfs之间进行数据迁移的java简单案例

1.开发环境
windows
intellij idea2017
hadoop-2.9.1
mysql-5.7.21
sqoop-1.4.7
2.windows安装mysql-5.7.21
mysql下载路径
https://cdn.mysql.com/archives/mysql-installer/mysql-installer-community-5.7.21.0.msi
双击运行
2.1 如果出现下图内容

请下载并运行
https://download.microsoft.com/download/B/A/4/BA4A7E71-2906-4B2D-A0E1-80CF16844F5F/dotNetFx45_Full_setup.exe
然后再安装mysql
2.2 可依照下列图进行安装和配置












3.windows安装hadoop-2.9.1，请参照
https://blog.csdn.net/a781136776/article/details/80458513
4.利用intellij idea 创建maven项目
4.1 新建project

4.2 选择quickstart

4.3 填写GroupId和ArtifactId

4.4 项目详细

4.5 填写项目的名称和保存路径

5.代码
5.1 pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0modelVersion>

  <groupId>sqoopgroupId>
  <artifactId>utilartifactId>
  <version>1.0-SNAPSHOTversion>

  <name>utilname>
  
  <url>http://www.example.comurl>

  <properties>
    <project.build.sourceEncoding>UTF-8project.build.sourceEncoding>
    <maven.compiler.source>1.7maven.compiler.source>
    <maven.compiler.target>1.7maven.compiler.target>
  properties>

  <dependencies>
    <dependency>
      <groupId>junitgroupId>
      <artifactId>junitartifactId>
      <version>4.11version>
      <scope>testscope>
    dependency>
    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-commonartifactId>
      <version>2.9.0version>
      <scope>providedscope>
    dependency>
    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-mapreduce-client-coreartifactId>
      <version>2.9.0version>
    dependency>
    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-hdfsartifactId>
      <version>2.9.0version>
    dependency>
    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-mapreduce-client-commonartifactId>
      <version>2.9.0version>
    dependency>
    <dependency>
      <groupId>org.apache.avrogroupId>
      <artifactId>avroartifactId>
      <version>1.8.0version>
    dependency>
    <dependency>
      <groupId>org.apache.sqoopgroupId>
      <artifactId>sqoopartifactId>
      <version>1.4.7version>
    dependency>
    <dependency>
      <groupId>mysqlgroupId>
      <artifactId>mysql-connector-javaartifactId>
      <version>5.1.36version>
    dependency>
  dependencies>
project>

5.2 SqoopTest.java

package sqoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.Sqoop;
import org.apache.sqoop.tool.SqoopTool;
import org.apache.sqoop.util.OptionsFileUtil;

public class SqoopTest {
    private static int importDataFromMysql() throws Exception {
        String[] args = new String[] {
                "--connect","jdbc:mysql://127.0.0.1:3306/test",
                "--driver","com.mysql.jdbc.Driver",
                "--username","root",
                "--password","root",
                "--table","log",
                "--delete-target-dir",
                "--num-mappers","1",
                "--target-dir","/test/log"           
        };
        String[] expandArguments = OptionsFileUtil.expandArguments(args);

        SqoopTool tool = SqoopTool.getTool("import");
        Configuration conf = new Configuration();
        conf.set("fs.default.name", "hdfs://127.0.0.1:9000");//设置HDFS服务地址
        Configuration loadPlugins = SqoopTool.loadPlugins(conf);
        Sqoop sqoop = new Sqoop((com.cloudera.sqoop.tool.SqoopTool) tool, loadPlugins);
        return Sqoop.runSqoop(sqoop, expandArguments);
    }

    public static void main(String[] args) throws Exception {
        Long t1=System.currentTimeMillis();
        importDataFromMysql();
        System.out.println(System.currentTimeMillis()-t1);
    }
}

6.在mysql中准备数据

create database test;

use test;  

create table log(id int not null,time_stamp varchar(40),category varchar(40),type varchar(40),servername varchar(40),code varchar(40),msg varchar(40),primary key (id));  

insert into log(id,time_stamp,category,type,servername,code,msg) values(1,'apr-8-2014-7:06:16-pm-pdt','notice','weblogicserver','adminserver','bea-000365','server state changed to standby');  
insert into log(id,time_stamp,category,type,servername,code,msg) values(2,'apr-8-2014-7:06:17-pm-pdt','notice','weblogicserver','adminserver','bea-000365','server state changed to starting');  
insert into log(id,time_stamp,category,type,servername,code,msg) values(3,'apr-8-2014-7:06:18-pm-pdt','notice','weblogicserver','adminserver','bea-000365','server state changed to admin');  
insert into log(id,time_stamp,category,type,servername,code,msg) values(4,'apr-8-2014-7:06:19-pm-pdt','notice','weblogicserver','adminserver','bea-000365','server state changed to resuming');  
insert into log(id,time_stamp,category,type,servername,code,msg) values(5,'apr-8-2014-7:06:20-pm-pdt','notice','weblogicserver','adminserver','bea-000361','started weblogic adminserver');  
insert into log(id,time_stamp,category,type,servername,code,msg) values(6,'apr-8-2014-7:06:21-pm-pdt','notice','weblogicserver','adminserver','bea-000365','server state changed to running');  
insert into log(id,time_stamp,category,type,servername,code,msg) values(7,'apr-8-2014-7:06:22-pm-pdt','notice','weblogicserver','adminserver','bea-000360','server started in running mode');  
commit;  

7.maven库中不包括sqoop-1.4.7.jar，所以请下载sqoop-1.4.7
https://archive.apache.org/dist/sqoop/1.4.7/sqoop-1.4.7.tar.gz
解压后将sqoop-1.4.7.bin__hadoop-2.6目录下的sqoop-1.4.7.jar复制到你的maven本地库，
如C:\Users\Administrator\.m2\repository\org\apache\sqoop\sqoop\1.4.7
8.运行
9.sqooputil.zip
https://download.csdn.net/download/a781136776/10438741