Hadoop HDFS文件系统通过java FileSystem 实现上传下载等

package linlintest;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class HdfsFileSystem {
	
	 public static void main(String[] args) throws Exception  {
    	 //String uri="hdfs://LL-167:8020/";   //hdfs 地址
		 // String remote="hdfs://LL-167:8020/lin/in1/1.txt"; //hdfs上的路径
		 String uri="hdfs://192.168.0.151:8020/";   //hdfs 地址
         String local="C:/Users/Administrator/Desktop/a.txt";  //本地路径
         String remote="hdfs://192.168.0.151:8020/Workspace/houlinlin";
         Configuration conf = new Configuration();
         
         //cat(conf ,uri,"hdfs://LL-167:8020/lin/in1/1.txt");
         //download(conf ,uri,remote,local);
        //  delete(conf ,uri,"hdfs://192.168.0.173:8020/Workspace/houlinlin");
         // markDir(conf ,uri,"hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");
        
        //  checkDir(uri,"d:/file");
        // getFile(conf ,uri,"","hdfs://192.168.0.151:8020/Workspace/houlinlin/a.txt");
         copyFile(conf,uri,"C:/Users/Administrator/Desktop/8a8380c745d05d370145d06719aa3c89.txt","hdfs://192.168.0.151:8020/Workspace/houlinlin/file/apply/2014/8a8380c7459dd8b90145a1fafb500235");
       //   ls(conf ,"hdfs://fulonghadoop","hdfs://fulonghadoop/");
    }
    /**
     * 上传文件
     * @param conf
     * @param local
     * @param remote
     * @throws IOException
     */
	public static void copyFile(Configuration conf , String uri , String local, String remote) throws IOException {
		FileSystem fs = FileSystem.get(URI.create(uri), conf);
		fs.copyFromLocalFile(new Path(local), new Path(remote));
		System.out.println("copy from: " + local + " to " + remote);
		fs.close();
	}
	
	/**
	 * 获取hdfs上文件流
	 * @param conf
	 * @param uri
	 * @param local
	 * @param remote
	 * @throws IOException
	 */
	public static  void getFileStream(Configuration conf , String uri , String local, String remote) throws IOException{
		FileSystem fs = FileSystem.get(URI.create(uri), conf);
		Path path= new Path(remote);
		FSDataInputStream in = fs.open(path);//获取文件流
		FileOutputStream fos = new FileOutputStream("C:/Users/Administrator/Desktop/b.txt");//输出流
		int ch = 0;
		while((ch=in.read()) != -1){  
            fos.write(ch);  
        }  
		System.out.println("-----");
		in.close(); 
		fos.close();  
	}
	
	/**
	 * 创建文件夹
	 * @param conf
	 * @param uri
	 * @param remoteFile
	 * @throws IOException
	 */
	public static void markDir(Configuration conf , String uri , String remoteFile ) throws IOException{
		FileSystem fs = FileSystem.get(URI.create(uri), conf);
		Path path = new Path(remoteFile);
		
		fs.mkdirs(path);
		System.out.println("创建文件夹"+remoteFile);
		 
	}
	/**
	 * 查看文件
	 * @param conf
	 * @param uri
	 * @param remoteFile
	 * @throws IOException
	 */
	 public static void cat(Configuration conf , String uri ,String remoteFile) throws IOException {
			Path path = new Path(remoteFile);
			FileSystem fs = FileSystem.get(URI.create(uri), conf);
			FSDataInputStream fsdis = null;
			System.out.println("cat: " + remoteFile);
			try {
				fsdis = fs.open(path);
				IOUtils.copyBytes(fsdis, System.out, 4096, false);
			} finally {
				IOUtils.closeStream(fsdis);
				fs.close();
			}
	}
	 /**
	  * 下载 hdfs上的文件
	  * @param conf
	  * @param uri
	  * @param remote
	  * @param local
	  * @throws IOException
	  */
	 public static void download(Configuration conf , String uri ,String remote, String local) throws IOException {
			Path path = new Path(remote);
			FileSystem fs = FileSystem.get(URI.create(uri), conf);
			fs.copyToLocalFile(path, new Path(local));
			System.out.println("download: from" + remote + " to " + local);
			fs.close();
	}
	 /**
	  * 删除文件或者文件夹
	  * @param conf
	  * @param uri
	  * @param filePath
	  * @throws IOException
	  */
	 public static void delete(Configuration conf , String uri,String filePath) throws IOException {
			Path path = new Path(filePath);
			FileSystem fs = FileSystem.get(URI.create(uri), conf);
			fs.deleteOnExit(path);
			System.out.println("Delete: " + filePath);
			fs.close();
	}
	 /**
	  * 查看目录下面的文件
	  * @param conf
	  * @param uri
	  * @param folder
	  * @throws IOException
	  */
	 public static  void ls(Configuration conf , String uri , String folder) throws IOException {
			Path path = new Path(folder);
			FileSystem fs = FileSystem.get(URI.create(uri), conf);
			FileStatus[] list = fs.listStatus(path);
			System.out.println("ls: " + folder);
			System.out.println("==========================================================");
			for (FileStatus f : list) {
				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(),f.isDirectory() , f.getLen());
			}
			System.out
					.println("==========================================================");
			fs.close();
	}

		/**
		 * 
		 * @param parentName 绝对路径地址
		 * @throws Exception
		 */
		public static void checkDir(String uri,String parentName)	throws Exception{
			//D:\file
			Configuration conf = new Configuration();
			File file = new File(parentName);
			boolean flag = true;
			while (flag)	{
				//查出parentName下的所有文件
				File[] fileNames = file.listFiles(new FileFilter());
				if(fileNames != null)	{
					for (int i = 0; i < fileNames.length; i++) {
						File f = fileNames[i];
						//System.out.println("parent directory:"+f.getParent()+",file name:"+f.getName());
						System.out.println("parent directory:"+f.getParent().replace("\\", "/").substring(2)+",file name:"+f.getName());
						String remoteFolrd= "hdfs://192.168.0.173:8020/Workspace/houlinlin"+f.getParent().replace("\\", "/").substring(2);
				        markDir(conf ,uri,remoteFolrd);
 				        copyFile(conf ,uri,f.getParent()+"\\"+f.getName(),remoteFolrd);
					}				
				}
				//查出parentName下的所有目录
				File[] directories = file.listFiles(new DirectortyFilter());
				if(directories != null)	{
					for (int i = 0; i < directories.length; i++) {
						File dir = directories[i];
						//绝对路径
						String path =  dir.getAbsolutePath();
						//递归
						checkDir(uri,path);
					}
				}
				flag = false;
			}
		}
	 
}