实训笔记7.13

7.13

一、座右铭

我的故事你说，我的文字我落，我值几两你定，我去何方我挑。

二、Hadoop的安装配置

九个核心文件的修改

三、Hadoop的HDFS的基本操作

3.1 HDFS的组成

NameNode

DataNode

SecondaryNameNode

3.2 HDFS的命令行操作方式

hdfs dfs -xxxx xxxxx

3.3 HDFS的JavaAPI操作方式

FileSystem类

四、代码示例

package com.sxuek;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * HDFS查看某个路径下的所有文件和文件夹的信息
 */
public class Demo {
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.68.101:9000"), conf, "root");
        /**
         *
         */
        FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
        for (FileStatus fileStatus : fileStatuses) {
            System.out.println(fileStatus.getPath());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getOwner());
            System.out.println(fileStatus.getGroup());
        }

    }
}

package com.sxuek;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * HDFS的相关判断类型的JavaAPI操作
 */
public class Demo01 {
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.68.101:9000"), conf, "root");
        boolean b = fs.isDirectory(new Path("/demo"));
        System.out.println(b);
        boolean b1 = fs.isFile(new Path("/demo"));
        System.out.println(b1);

        boolean exists = fs.exists(new Path("/a"));
        System.out.println(exists);
    }
}

package com.sxuek;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * HDFS的创建相关的JavaAPI
 */
public class Demo02 {
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.68.101:9000"), conf, "root");
        boolean mkdirs = fs.mkdirs(new Path("/a/b"));
        System.out.println(mkdirs);

        boolean newFile = fs.createNewFile(new Path("/a/a.txt"));
        System.out.println(newFile);

    }
}

package com.sxuek;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * HDFS提供了一个可以借助JavaIO流读取数据的方法
 */
public class Demo03 {
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.68.101:9000"), conf, "root");

        FSDataInputStream inputStream = fs.open(new Path("/jdk-8u371-linux-x64.tar.gz"));
        inputStream.seek(128*1024*1024);
        FileOutputStream fos = new FileOutputStream("c:/users/lenovo/desktop/block2");
        int read = 0;
        while ((read = inputStream.read()) != -1){
            fos.write(read);
        }
        System.out.println("第二个数据块读取完成");
    }
}

log4j.rootLogger=INFO,Console

# Console - 将日志打印到控制台
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n