Hadoop2.6 API文件操作总结

Hadoop dfs文件操作API官方文档

• 当前稳定版api
• Hadoop 2.6.5 DFS API

连接配置

Configuration conf = new Configuration();
conf.set("fs.defaultFS", "192.168.2.154:8020");

显示指定路径下的文件（类似 hdfs dfs -ls）

public void list(Configuration conf) throws IOException{
    FileSystem fs = FileSystem.get(conf);
    Path ph = new Path("/data/");
    FileStatus[] files = fs.listStatus(ph);
    for(FileStatus file : files) {
        if (file.isDirectory()) {
            System.out.println("Directory:" + file.getPath().toString());
        }
        
        if (file.isFile()) {
            System.out.println("File:" + file.getPath().toString());
        }
    }

    fs.close();
}

创建文件夹（类似 hdfs dfs -mkdir）

public void mkdir(Configuration conf) throws IOException{
    FileSystem fs = FileSystem.get(conf);
    Path ph =  new Path("/data/new_dir");
    fs.mkdirs(ph);
    fs.close();
}

删除文件夹（类似 hdfs dfs -rmdir）

public void rmdir(Configuration conf) throws IOException{
    FileSystem fs = FileSystem.get(conf);
    Path ph =  new Path("/data/new_dir");
    fs.delete(ph);
    fs.close();
}

• fs.delete(ph); 删除文件或者空的文件夹
• fs.delete(ph, true); 删除非空目录及其中的内容；

从本地拷贝文件到dfs（类似 hdfs dfs -copyFromLocal）

public void copyFromLocal(Configuration conf) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path fromPath = new Path("C:/Users/lenovo/Desktop/SequenceOutput_lib/activation-1.1.jar");
    Path toPath = new Path("/data/");
    fs.copyFromLocalFile(fromPath, toPath);
    fs.close();
}

按行读取文件内容

public void readStream(Configuration conf) throws IOException{
    FileSystem fs = FileSystem.get(conf);
    Path file =  new Path("/data/user_login.txt");
    FSDataInputStream is = fs.open(file);
    BufferedReader br = new BufferedReader(new InputStreamReader(is,"utf-8"));
    String line = "";
    while( (line=br.readLine()) != null) {
        System.out.println(line);
    }
    
    br.close();
    is.close();
    fs.close();
}

写文件

public void writeStream(Configuration conf) throws IOException{
    FileSystem fs = FileSystem.get(conf);
    Path file =  new Path("/data/new_user_login.txt");
    FSDataOutputStream os = fs.create(file);
    BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os,"utf-8"));
    
    String line = "2017-09-08";
    bw.write(line);
    bw.newLine();
    
    bw.close();
    os.close();
    fs.close();
}

读序列文件

public void readSequenceFile(Configuration conf) throws IOException{
    FileSystem fs = FileSystem.get(conf);
    Path file =  new Path("/output/login/part-m-00000");
    SequenceFile.Reader rd = new SequenceFile.Reader(fs, file, conf);
    Text key = new Text();
    Text val = new Text();
    
    while(rd.next(key, val)) {
        System.out.println(key.toString() + "=>" + val.toString());
    }
    
    System.out.println("helloxiaojie");
    fs.close();
}