MapReduce的文件操作

读取SequenceFile文件

package com.cxw.read;

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;

public class SequenceFileReadFile {

	public static void main(String[] args) {
		SequenceFile.Reader reader = null;
		try {
			String uri = "你想要读取的SequenceFile所在位置";
			Configuration conf = new Configuration();
			FileSystem fs = FileSystem.get(URI.create(uri),conf);
			Path path = new Path(uri);
			reader = new SequenceFile.Reader(fs, path, conf);
			Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
			Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
			long position = reader.getPosition();
			while(reader.next(key,value)){
				String syncSeen = reader.syncSeen()?"*":"";
				System.out.printf("[%s%s]\t%s\n",position,syncSeen,key,value);
				position = reader.getPosition();//beginning of next record
			}
		} catch (IOException e) {
			e.printStackTrace();
		}finally{
			IOUtils.closeStream(reader);
		}
		
	}

}

 

写SequenceFile文件

package com.cxw.write;

import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;

public class SequenceFileWriteDemo {
	private static String[] myValue = {"hello world","bye world","hello hadoop","bye hadoop"};
	public static void main(String[] args) {
		SequenceFile.Writer writer = null;
		try {
			String uri = "你想要生成的SequenceFile的位置：";
			Configuration conf = new Configuration();
				FileSystem fs = FileSystem.get(URI.create(uri),conf);
			Path path = new Path(uri);
			IntWritable key = new IntWritable();
			Text value = new Text();
			writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
			for (int i = 0; i < 5000000; i++) {
				key.set(5000000-i);
				value.set(myValue[i%myValue.length]);
				writer.append(key, value);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}finally{
			IOUtils.closeStream(writer);
		}
	}
}