MR中使用sequnceFIle输入文件

转换原始数据为块压缩的SequenceFIle

 

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.SequenceFile.CompressionType;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.io.compress.GzipCodec;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;



import com.hadoop.compression.lzo.LzoCodec;



public class ToSeqFile extends Configured implements Tool {

    @Override

    public int run(String[] arg0) throws Exception {

        Job job = new Job();

        job.setJarByClass(getClass());

        Configuration conf=getConf();

        FileSystem fs = FileSystem.get(conf);

    

        FileInputFormat.setInputPaths(job, "/home/hadoop/tmp/tmplzo.txt");

        Path outDir=new Path("/home/hadoop/tmp/tmplzo.out");

        fs.delete(outDir,true);

        FileOutputFormat.setOutputPath(job, outDir);

        

        //job.setMapperClass(IndentityMapper);

        job.setNumReduceTasks(0);

        job.setOutputKeyClass(LongWritable.class);

        job.setOutputValueClass(Text.class);

        //设置OutputFormat为SequenceFileOutputFormat

        job.setOutputFormatClass(SequenceFileOutputFormat.class);

        //允许压缩

         SequenceFileOutputFormat.setCompressOutput(job, true);

         //压缩算法为gzip

         SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class);

        //压缩模式为BLOCK

         SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);





        return job.waitForCompletion(true)?0:1;

    }



    public static void main(String[] args) throws Exception {

        int res = ToolRunner.run(new Configuration(), new ToSeqFile(), args);

        System.exit(res);

    }

}

 

MR处理压缩后的sequenceFile

import org.apache.hadoop.io.Text;



import java.io.File;

import java.io.IOException;

import java.net.URI;

import java.util.Iterator;

import java.util.StringTokenizer;



import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.io.compress.*;

import org.apache.hadoop.mapreduce.ContextFactory;

import org.apache.hadoop.mapreduce.InputSplit;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;

import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import org.apache.hadoop.util.GenericOptionsParser;

import org.apache.hadoop.util.Progressable;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

//import org.apache.hadoop.mapred.DeprecatedLzoTextInputFormat;













import com.hadoop.compression.lzo.LzoCodec;

import com.hadoop.mapreduce.LzoTextInputFormat;



public class compress extends Configured implements Tool {

	private static final Log log = LogFactory.getLog(compress.class);



	private static class ProvinceMapper extends

			Mapper<Object, Text, Text, Text> {

		@Override

		protected void map(Object key, Text value, Context context)

				throws IOException, InterruptedException {

			//System.out.println(value);

			

			// InputSplit inputSplit = context.getInputSplit();

			//String fileName = ((FileSplit) inputSplit).getPath().toString();

			

			//System.out.println(fileName);

			context.write(value, value);

		}

	}



	private static class ProvinceReducer extends

			Reducer<Text, Text, Text, Text> {

		@Override

		protected void reduce(Text key, Iterable<Text> values, Context context)

				throws IOException, InterruptedException {

			for (Text va : values) {

				// System.out.println("reduce " + key);

				context.write(key, key);

			}

		}

	}



	public static void main(String[] args) throws Exception {

		ToolRunner.run(new Configuration(), new compress(), args);

	}



	public static final String REDUCES_PER_HOST = "mapreduce.sort.reducesperhost";



	@Override

	public int run(String[] args) throws Exception {

		log.info("我的服务查询开始.....................................");

        

		long beg = System.currentTimeMillis();

		int result = 0;

		Configuration conf = new Configuration();



		conf.set(

				"io.compression.codecs",

				"org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzopCodec");

		conf.set("io.compression.codec.lzo.class",

				"com.hadoop.compression.lzo.LzoCodec");

		

		

		conf.setBoolean("mapreduce.map.output.compress", true);

	    conf.setClass("mapreduce.map.output.compression.codec", SnappyCodec.class, CompressionCodec.class);

	   // conf.setBoolean("mapreduce.output.fileoutputformat.compress", true); // 是否压缩输出

	    conf.setClass("mapreduce.output.fileoutputformat.compress.codec", SnappyCodec.class, CompressionCodec.class);



		String[] argArray = new GenericOptionsParser(conf, args)

				.getRemainingArgs();



		if (argArray.length != 2) {

			System.err.println("Usage: compress <in> <out>");

			System.exit(1);

		}



		// Hadoop总共有5个Job.java

		// /hadoop-2.0.0-cdh4.5.0/src/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java

		Job job = new Job(conf, "compress");

		job.setJarByClass(compress.class);

		job.setMapperClass(ProvinceMapper.class);

		job.setReducerClass(ProvinceReducer.class);

		job.setMapOutputKeyClass(Text.class);

		job.setMapOutputValueClass(Text.class);

		job.setOutputKeyClass(Text.class);

		job.setOutputValueClass(Text.class);



		 //job.setInputFormatClass(LzoTextInputFormat.class); // TextInputFormat

		// MyFileinput

		

		// 使用lzo索引文件作为输入文件

		// job.setInputFormatClass(LzoTextInputFormat.class);

		job.setInputFormatClass(SequenceFileInputFormat.class);

		

		// SequenceFileOutputFormat.set(job, LzoCodec.class);

		

		// 测试块大小

		// FileInputFormat.setMinInputSplitSize(job, 150*1024*1024);

		// FileInputFormat.setMinInputSplitSize(job, 301349250);

		// FileInputFormat.setMaxInputSplitSize(job, 10000);



		// 推测执行的开关 另外还有针对map和reduce的对应开关

		// job.setSpeculativeExecution(false);

		FileInputFormat.addInputPath(job, new Path(argArray[0]));

		FileOutputFormat.setOutputPath(job, new Path(argArray[1]));



		String uri = argArray[1];

		Path path = new Path(uri);

		FileSystem fs = FileSystem.get(URI.create(uri), conf);

		if (fs.exists(path)) {

			fs.delete(path);

		}



		result = job.waitForCompletion(true) ? 0 : 1;

		

//		try {

//			result = job.waitForCompletion(true) ? 0 : 1;

//		} catch (ClassNotFoundException | InterruptedException e) {

//			e.printStackTrace();

//		}

		long end = (System.currentTimeMillis() -beg) ;

        System.out.println("耗时:" + end);

		return result;

	}

}

 

测试结果

文件大小 544M(未使用任何压缩)
耗时:73805

使用 seqencefile(block使用lzo压缩, 中间结果使用snappy压缩)

44207s