Hive之——用户自定义聚合函数

转载请注明出处：https://blog.csdn.net/l1028386804/article/details/88536189

基于UDAF执行的转换的不同，在不同阶段的返回值类型也可能是不同的。
在写UDAF的时候一定要注意内存使用的问题。通过配置参数mapred.child.java.opts可以调整执行过程的内存需求量，但是这种方式并非总是奏效:

	mapred.child.java.opts
	-Xmx200m

创建一个COLLECT UDAF来模拟GROUP_CONCAT
MySQL中有一个非常有用的函数名为GROUP_CONCAT，其可以将一组中的所有元素按照用户指定的分隔符组装成一个字符串。
GROUP_CONCAT在MySQL中的用法如下:

mysql> create table people(name string, friendname string);

mysql> select * from people;
bob		sara
bob		john
bob		ted
john	bob
ted		sara

mysql> select name, group_concat(friendname separator ',') from people group by name;

我们无需增加新的语法就可以在Hive中实现同样的转换。
接下来我们创建GenericUDAFCollect类和GenericUDAFMkListEvaluator类

package com.lyz.hadoop.hive.udaf;

import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

/**
 *  自定义聚合函数
 * @author liuyazhuang
 *
 */
@Description(name="collect", 
value = "_FUNC_(x) - Returns a list of objects. CAUTION will easily OOM on large data sets ")
public class GenericUDAFCollect extends AbstractGenericUDAFResolver {
	public GenericUDAFCollect() {
		
	}
	
	@Override
	public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticException {
		if(parameters.length != 1) {
			throw new UDFArgumentTypeException(parameters.length - 1, "Exactly one argument is expected.");
		}
		if(parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
			throw new UDFArgumentTypeException(0, "Pnly primitive type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1.");
		}
		return new GenericUDAFMkListEvaluator(); 
	}
}

package com.lyz.hadoop.hive.udaf;

import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;

/**
   * 自定义聚合函数
 * @author liuyazhuang
 *
 */
public class GenericUDAFMkListEvaluator extends GenericUDAFEvaluator {
	private PrimitiveObjectInspector inputOI;
	private StandardListObjectInspector loi;
	private StandardListObjectInspector internalMergeOI;
	
	@Override
	public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException {
		super.init(m, parameters);
		if(m == Mode.PARTIAL1) {
			inputOI = (PrimitiveObjectInspector) parameters[0];
			return ObjectInspectorFactory.getStandardListObjectInspector((PrimitiveObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(inputOI));
		}else {
			if(!(parameters[0] instanceof StandardListObjectInspector)) {
				inputOI = (PrimitiveObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
				return (StandardListObjectInspector) ObjectInspectorFactory.getStandardListObjectInspector(inputOI);
			}else {
				internalMergeOI = (StandardListObjectInspector) parameters[0];
				inputOI = (PrimitiveObjectInspector) internalMergeOI.getListElementObjectInspector();
				loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
				return loi;
			}
		}
	}
	static class MkArrayAggregationBuffer implements AggregationBuffer{
		List