java 读取 hadoop 的文件 demo

是什么？

      使用java 读取 hadoop 中的一个文件。

 

为什么？

     用于初步接触hadoop开发。

 

怎么做？

一、依赖jar

    4.0.0

    myHadoop
    myHadoop
    1.0-SNAPSHOT

    
        2.4.0
        2.4.0
        2.4.0
        2.4.0
        2.4.0
        3.3.2
        2.4
    

    
        
            org.apache.hadoop
            hadoop-hdfs
            ${hadoop.hdfs.version}
        
        
            org.apache.hadoop
            hadoop-mapreduce-client-core
            ${hadoop.mapreduce.client.core.version}
        
        
            org.apache.hadoop
            hadoop-common
            ${hadoop.common.version}
        
        
            org.apache.hadoop
            hadoop-mapreduce-client-common
            ${hadoop.mapreduce.client.common.version}
        
        
            org.apache.hadoop
            hadoop-mapreduce-client-jobclient
            ${hadoop.mapreduce.client.jobclient.version}
        
        
            org.apache.commons
            commons-lang3
            ${commons.lang.version}
        
        
            commons-io
            commons-io
            ${commons.io.version}
        
    

    
        gQuery-${version}
        
        
            
                org.apache.maven.plugins
                maven-compiler-plugin
                2.3.1
                
                    1.7
                    1.7
                    utf8
                
            
            
            
                org.apache.maven.plugins
                maven-jar-plugin
                2.4
                
                
                    
                    
                        
                        false

                        
                        
                            
                            true
                            
                            lib/
                            
                            com.geotmt.hadoop.App
                        
                        
                        
                            .
                        
                    
                    
                    
                        **/*.properties
                        **/*.xml
                    
                
            

            
                org.apache.maven.plugins
                maven-resources-plugin
                3.1.0
                
                    
                        copy-resources
                        package
                        
                            copy-resources
                        
                        
                            UTF-8
                            
                            ${project.build.directory}/etc
                            
                                
                                
                                    
                                    src/main/resources/
                                    
                                    
                                        **/*.properties
                                        **/*.xml
                                    
                                    
                                    true
                                
                            
                        
                    
                
            

            
            
                org.apache.maven.plugins
                maven-dependency-plugin
                
                
                    
                    
                        
                        copy-dependencies
                        
                        package
                        
                        
                            copy-dependencies
                        
                        
                            
                            
                                ${project.build.directory}/etc/lib
                            
                            false
                            false
                            true
                            junit
                        
                    
                
            
        
    

java代码

package com.geotmt.hadoop.hdfs;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.InputStream;
import java.net.URI;

/**
 * 从Hdfs读取一个文件
 *
 * Created by c.z on 2018/8/10. */
public class ReadFileHandle {

    public static void main(String[] args) throws Exception {
        ReadFileHandle readFileHandle = new ReadFileHandle();
        readFileHandle.readFile();
    }
    
    public void readFile() throws Exception{
        String uri = "hdfs://10.111.32.142:8020/user/zhaochao/20180727/sms.1532681090912.log.tmp";
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(uri),conf);
        try(InputStream in = fs.open(new Path(uri)))
        {
            IOUtils.copy(in,System.out);
        }
    }
}