Java操作HDFS

一、读取hdfs文件数据

方法一：使用hadoop URL读取数据

  URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
    	  InputStream in = null;
    	  
    	  try {
			in = new URL("hdfs://172.16.169.10:9000/WordCount.txt").openStream();
			IOUtils.copyBytes(in, System.out, 4096, false);
		} catch (IOException e) {
			
			e.printStackTrace();
		}finally {
			IOUtils.closeStream(in);
		}
    	  

方法二：使用FileSystem API读取数据

 String url = args[0];
    	   Configuration conf = new Configuration();
    	   conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
           FileSystem fs  = FileSystem.get(URI.create(url), conf);
           InputStream in = null;
          
			try {
				in = fs.open(new Path(url));
				IOUtils.copyBytes(in, System.out, 4096, false);
			} finally {
				IOUtils.closeStream(in);
			}
           

二、创建、删除文件（夹）

        Configuration conf = new Configuration();

     	FileSystem fs = FileSystem.get(new URI("hdfs://172.16.169.10:9000"), conf);
//    	crate a file floder
    	boolean boo = fs.mkdirs(new Path("/usr/fu/zi/Word.txt"));
    	System.out.println(boo);
//    	create a file
    	fs.create(new Path("/fu/zi/HelloWord.txt"));
//    	delete files
    	boolean boo1 = fs.delete(new Path("/fu"), true);
    	System.out.println(boo1);
    	boolean boo2 = fs.delete(new Path("/usr"), true);
    	System.out.println(boo2);
    	

三、向文件中写入数据

 

四、maven相关依赖

  
      
     
        
        org.apache.maven.plugins  
        maven-compiler-plugin  
          
          1.8  
          1.8 
          UTF-8 
          
      
       
    
    
    
    
  
    
    org.apache.hadoop
    hadoop-common
    2.7.4





    com.google.guava
    guava
    25.0-jre


 
            jdk.tools
            jdk.tools
            1.8
            system
            ${JAVA_HOME}/lib/tools.jar
    
    
    
    commons-logging
    commons-logging
    1.1.3

  


    org.apache.hadoop
    hadoop-hdfs
    2.7.4