Java 读写 hdfs文件或者目录

1.读取单个文件

Date date = DateUtil.getSpecifiedDayBefore();
String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd");
String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday;

 Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(path), conf);
FSDataInputStream hdfsInStream = fs.open(new Path(path));
InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
BufferedReader br = new BufferedReader(isr);
String line;
// int k = 0;
while ((line = br.readLine()) != null) {
    System.out.println(line);
}

2.读取文件夹

Date date = DateUtil.getSpecifiedDayBefore();
String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd");
String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday;

Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(path), conf);
FileStatus[] status = fs.listStatus(new Path(path));
for (FileStatus file : status) {
    if (!file.getPath().getName().startsWith("newsMap")) {
              continue;
          }
    FSDataInputStream hdfsInStream = fs.open(file.getPath());
    InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
    BufferedReader br = new BufferedReader(isr);
    String line;
    // int k = 0;
    while ((line = br.readLine()) != null) {
        System.out.println(line);
    }

}