eclipse通过hdfs提供的api对hdfs进行文件操作
上一文介绍了在本机windows搭建一个伪分布式环境，接下来就是在eclipse进行hdfs文件的读写等操作。
第一步需要获取eclipse进行hadoop开发的插件支持，hadoop第一代和第二代不同，第二代貌似没直接给插件，需要网上找，第一代的有，怎样去获取呢？想必大家很关心，我开始也纳闷这玩意儿在哪里？在网上也找过放进去结果没用，不知道是什么问题。后来查资料发现第一代的安装包有给。
拿到jar包找到eclipse的安装目录下的plugins文件夹下面，然后重启eclipse，接下来打开就发现有个蓝色大象，你可以在eclipse直接看hdfs上面的文件，点击新建一个连接
接下来写代码进行操作
package com.qqw.hadoop;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
public class HdfsUtil {
private Configuration conf = new Configuration();
private FileSystem coreSys=null;
private String rootPath=new String("hdfs://localhost:9000/");
/**
* 在HDFS上创建文件目录
* @throws IOException */
public void createDirOnHDFS() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path demoDir=new Path(rootPath+"demoDir");
boolean isSuccess=true;
try {
isSuccess=coreSys.mkdirs(demoDir);
} catch (IOException e) {
isSuccess=false;
}
System.out.println(isSuccess?"目录创建成功！":"目录创建失败!");
}
/**
* 在hdfs上创建文件
* @throws IOException
*/
public void createFileOnHDFS() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path hdfsPath = new Path(rootPath + "demoDir/createDemoFile1.txt");
System.out.println(coreSys.getHomeDirectory());
String content = "Hello hadoop,this is first time that I create file on hdfs";
FSDataOutputStream fsout = coreSys.create(hdfsPath);
BufferedOutputStream bout = new BufferedOutputStream(fsout);
bout.write(content.getBytes(), 0, content.getBytes().length);
bout.close();
fsout.close();
System.out.println("文件创建完毕！");
}
/**
* 从hdfs上下载文件到本地
* @throws IOException
*/
public void downLoadFromHdfs() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path hdfsPath = new Path(rootPath + "demoDir/createDemoFile1.txt");
FSDataInputStream hdfsInStream = coreSys.open(hdfsPath);
OutputStream out = new FileOutputStream("D:\\createDemoFile1.txt");
byte[] ioBuffer = new byte[1024];
int readLen = hdfsInStream.read(ioBuffer);
while(-1 != readLen){
out.write(ioBuffer, 0, readLen);
readLen = hdfsInStream.read(ioBuffer);
}
out.close();
hdfsInStream.close();
coreSys.close();
}
/**
* 上传文件到hdfs
* @throws IOException
*/
public void upLoadFromHdfs() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path remotePath=new Path(rootPath+"demoDir/");
coreSys.copyFromLocalFile(new Path("D:\\qq.txt"), remotePath);
System.out.println("Upload to:"+conf.get("fs.default.name"));
FileStatus [] files=coreSys.listStatus(remotePath);
for(FileStatus file:files){
System.out.println(file.getPath().toString());
}
}
/**
* 重命名文件
* @throws IOException
*/
public void rename() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path oldFileName=new Path(rootPath+"demoDir/createDemoFile1.txt");
Path newFileName=new Path(rootPath+"demoDir/newDemoFile.txt");
boolean isSuccess=true;
try {
isSuccess=coreSys.rename(oldFileName, newFileName);
} catch (IOException e) {
isSuccess=false;
}
System.out.println(isSuccess?"重命名成功！":"重命名失败！");
}
/**
* 删除文件
* @throws IOException
*/
public void deleteFile() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path deleteFile=new Path(rootPath+"demoDir/qq.txt");
boolean isSuccess=true;
try {
isSuccess=coreSys.delete(deleteFile, false);
} catch (IOException e) {
isSuccess=false;
}
System.out.println(isSuccess?"删除成功!":"删除失败！");
}
/**
* 查看文件是否存在
* @throws IOException
*/
public void exitFile() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path checkFile=new Path(rootPath+"demoDir/qq.txt");
boolean isSuccess=true;
try {
isSuccess=coreSys.exists(checkFile);
} catch (IOException e) {
isSuccess=false;
}
System.out.println(isSuccess?"存在!":"不存在！");
}
/**
* 获取某个路径下的全部文件
* @throws IOException
*/
public void allFile() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path targetDir=new Path(rootPath+"demoDir/");
FileStatus [] files=coreSys.listStatus(targetDir);
for(FileStatus file:files){
System.out.println(file.getPath()+"--"+file.getGroup()+"--"+file.getBlockSize()
+"--"+file.getLen()+"--"+file.getModificationTime()+"--"+file.getOwner());
}
}
/** *
* 查看某个文件在HDFS集群的位置
* @throws IOException
*/
public void findLocationOnHadoop() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
Path targetFile=new Path(rootPath+"demoDir/qq.txt");
FileStatus fileStaus=coreSys.getFileStatus(targetFile);
BlockLocation []bloLocations=coreSys.getFileBlockLocations(fileStaus, 0, fileStaus.getLen());
for(int i=0;i<bloLocations.length;i++){
System.out.println("block_"+i+"_location:"+bloLocations[i].getHosts()[0]);
}
}
/**
* 获取集群上结点的信息
* @throws IOException
*/
public void getNodeMsgHdfs() throws IOException{
coreSys=FileSystem.get(URI.create(rootPath), conf);
DistributedFileSystem distributedFileSystem=(DistributedFileSystem) coreSys;
DatanodeInfo []dataInfos=distributedFileSystem.getDataNodeStats();
for(int j=0;j<dataInfos.length;j++){
System.out.println("DataNode_"+j+"_Name:"+dataInfos[j].getHostName()+"--->"+dataInfos[j].getDatanodeReport()+"-->"+
dataInfos[j].getDfsUsedPercent()+"-->"+dataInfos[j].getLevel());
}
}
}
然后可以写个main方法或junit测试，用hadoop跑，可以在eclipse上或访问网页http://localhost:50070实时参看hdfs上文件。