hadoop mr实现单表列转行--mr system.out数据位置

阅读更多

 

 

 

 

1 代码和业务：

 

package mapreduce;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;



/**
a	b	1 
a	b   2 
a	b	3 
c	d	4 
c	d	5 
c	d	6

期待变成
a	b	1,2,3 
c	d	4,5,6 

 * @author zm
 * 
 */
public class ConcatWSMapReduce {

	public static class ConcatWSMapper extends Mapper{
		/**
		 * 每一行执行一次map函数
		 * @param key 表示字节在源文件中偏移量
		 * @param value 行文本内容
		 */
		
		protected void map(LongWritable key, Text value, Context context) throws java.io.IOException ,InterruptedException {
			final String[] splited = value.toString().split("\t");
			System.out.println("splited length is:  " + splited.length);
			String col1 = splited[0];
			String col2 = splited[1];
			String col3 = splited[2];
			System.out.println("col1: " + col1  +  "col2: " + col2 + "col3: " + col3);
			context.write(new ConcatWS(col1,col2), new Text(col3));
		};
	}
	
	
	//分组：【把相同key的value放到一起】    reduce方法是每一组调用一次 左侧结果 为3组 则调用3次reduce方法
	public static class ConcatWSReducer extends Reducer{
		/**
		 * 每个组调用一次reduce函数
		 * @param word 表示单词
		 * @param times 表示相同key的value的迭代器
		 */
		protected void reduce(ConcatWS ws, Iterable v2s, org.apache.hadoop.mapreduce.Reducer.Context context) throws java.io.IOException ,InterruptedException {
			StringBuilder  sb = new StringBuilder("");
			for (Text col3 : v2s) {
				sb.append(col3.toString()).append(",");
			}
			System.out.println("reduce key content:  " + ws.toString());
			System.out.println("reduce val content:  " + sb.toString());
			context.write(new Text(ws.toString()), new Text(sb.toString()));
		};
	}
	
	
	public static void main(String[] args) throws Exception {
		// 设置Job对象
		final Configuration conf = new Configuration();
		final Job job = new Job(conf);
		job.setJobName(ConcatWSMapReduce.class.getSimpleName());
		job.setJarByClass(ConcatWSMapReduce.class);
		// 给Job对象设置自定义 mapper  reducer
		job.setMapperClass(ConcatWSMapper.class);
		job.setReducerClass(ConcatWSReducer.class);
		// 设置map reduce输出参数类型
		job.setMapOutputKeyClass(ConcatWS.class);
		job.setMapOutputValueClass(Text.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
		// 设置Job任务要处理的数据源和输出数据目的地
		FileInputFormat.addInputPaths(job, "/zmdata/zm.txt"); // 注意是addInputPaths 用的是复数的方法
		Path outputpath = new Path("/zmdata/zmout");
		FileSystem fs = FileSystem.get(new URI("/"), conf);
		if(fs.exists(outputpath)){
			fs.delete(outputpath, true);
		}
		FileOutputFormat.setOutputPath(job, outputpath);
		// 执行Job
		job.waitForCompletion(true);
	}

	// 注意写成内部public 类 否则执行mr时 会报不识别ConcatWS.source
public static class ConcatWS implements WritableComparable{

			private String col1 = "";
			private String col2 = "";
			
			public ConcatWS(){}
			
			public ConcatWS(String col1, String col2){
				this.col1 = col1;
				this.col2 = col2;
			}
			
			@Override
			public void write(DataOutput out) throws IOException {
				out.writeUTF(col1);
				out.writeUTF(col2);
			}

			@Override
			public void readFields(DataInput in) throws IOException {
				this.col1 = in.readUTF();
				this.col2 = in.readUTF();
			}

			@Override
			public int compareTo(ConcatWS ws) {
				int result = 0;
				result = this.col1.compareTo(ws.col1);
				if(result == 0){
					result = this.col2.compareTo(ws.col2);
				}
				return result;
			}   
			
			 @Override  
		    public String toString() {   
		        return col1 + "\t" + col2 ;   
		    }  
			 
			
		}
	 
	 
}

 

 

 

 

2 本机使用ant脚本提交后执行，ant脚本如下：

 

	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
			
		
		
			
		
	

	
	
		
		
	
	
	
		
		
		
		清理完毕
	
	
	
		
		
	
	
	
		
			
			  
		
	

	
	
		
			
		
	
	
	
	
		
	
	
	
	      
	
	

 

 

 

3  执行的mr代码中写上了 sysotem.out 那么这些数据在哪里呢？  比如上面的输出：

 

 

上图所示，找到你执行mr任务的编号，进去后看如下图：

 

 

 

 

• 
• 大小: 8.6 KB

• 
• 大小: 30.7 KB

• 查看图片附件