Hive中UDF、UDAF和UDTF使用
1.Hive中的内置函数
org.apache.hadoop.hive.ql.exec.FunctionRegistry类中定义了Hive目前内置的自定义函数 registerGenericUDF("concat", GenericUDFConcat.class);
registerUDF("substr", UDFSubstr.class, false);
registerUDF("substring", UDFSubstr.class, false);
registerUDF("space", UDFSpace.class, false);
registerUDF("repeat", UDFRepeat.class, false);
registerUDF("ascii", UDFAscii.class, false);
registerGenericUDF("lpad", GenericUDFLpad.class);
registerGenericUDF("rpad", GenericUDFRpad.class);
registerUDF("ln", UDFLn.class, false);
registerUDF("log2", UDFLog2.class, false);
registerUDF("sin", UDFSin.class, false);
registerUDF("asin", UDFAsin.class, false);
registerUDF("cos", UDFCos.class, false);
registerUDF("acos", UDFAcos.class, false);
registerUDF("log10", UDFLog10.class, false);
registerUDF("log", UDFLog.class, false);
registerUDF("exp", UDFExp.class, false);
registerGenericUDF("power", GenericUDFPower.class);
registerGenericUDF("pow", GenericUDFPower.class);
registerUDF("sign", UDFSign.class, false);
registerUDF("pi", UDFPI.class, false);
registerUDF("degrees", UDFDegrees.class, false);
registerUDF("radians", UDFRadians.class, false);
registerUDF("atan", UDFAtan.class, false);
registerUDF("tan", UDFTan.class, false);
registerUDF("e", UDFE.class, false);
registerUDF("conv", UDFConv.class, false);
registerUDF("bin", UDFBin.class, false);
registerUDF("hex", UDFHex.class, false);
registerUDF("unhex", UDFUnhex.class, false);
registerUDF("base64", UDFBase64.class, false);
registerUDF("unbase64", UDFUnbase64.class, false);
registerGenericUDF("encode", GenericUDFEncode.class);
registerGenericUDF("decode", GenericUDFDecode.class);
registerGenericUDF("upper", GenericUDFUpper.class);
registerGenericUDF("lower", GenericUDFLower.class);
registerGenericUDF("ucase", GenericUDFUpper.class);
registerGenericUDF("lcase", GenericUDFLower.class);
registerGenericUDF("trim", GenericUDFTrim.class);
registerGenericUDF("ltrim", GenericUDFLTrim.class);
registerGenericUDF("rtrim", GenericUDFRTrim.class);
registerUDF("length", UDFLength.class, false);
registerUDF("reverse", UDFReverse.class, false);
registerGenericUDF("field", GenericUDFField.class);
registerUDF("find_in_set", UDFFindInSet.class, false);
registerUDF("like", UDFLike.class, true);
registerUDF("rlike", UDFRegExp.class, true);
registerUDF("regexp", UDFRegExp.class, true);
registerUDF("regexp_replace", UDFRegExpReplace.class, false);
registerUDF("regexp_extract", UDFRegExpExtract.class, false);
registerUDF("parse_url", UDFParseUrl.class, false);
registerGenericUDF("nvl", GenericUDFNvl.class);
registerGenericUDF("split", GenericUDFSplit.class);
registerGenericUDF("str_to_map", GenericUDFStringToMap.class);
registerGenericUDF("translate", GenericUDFTranslate.class);
registerGenericUDF("date_add", GenericUDFDateAdd.class);
registerGenericUDF("date_sub", GenericUDFDateSub.class);
registerGenericUDF("datediff", GenericUDFDateDiff.class);
registerUDF("get_json_object", UDFJson.class, false);
2.UDF
Hive的UDF开发只需要重构UDF类的evaluate函数即可。例：package hive.connect;
import org.apache.hadoop.hive.ql.exec.UDF;
public final class Add extends UDF {
public Integer evaluate(Integer a, Integer b) {
if (null == a || null == b) {
return null;
} return a + b;
}
public Double evaluate(Double a, Double b) {
if (a == null || b == null)
return null;
return a + b;
}
public Integer evaluate(Integer... a) {
int total = 0;
for (int i = 0; i < a.length; i++)
if (a[i] != null)
total += a[i];
return total;
}
}3.UDAF
1、一下两个包是必须的import org.apache.hadoop.hive.ql.exec.UDAF和 org.apache.hadoop.hive.ql.exec.UDAFEvaluator。
2、函数类需要继承UDAF类，内部类Evaluator实UDAFEvaluator接口。
3、Evaluator需要实现 init、iterate、terminatePartial、merge、terminate这几个函数。
a）init函数实现接口UDAFEvaluator的init函数。
b）iterate接收传入的参数，并进行内部的轮转。其返回类型为boolean。
c）terminatePartial无参数，其为iterate函数轮转结束后，返回轮转数据，terminatePartial类似于hadoop的Combiner。
d）merge接收terminatePartial的返回结果，进行数据merge操作，其返回类型为boolean。
e）terminate返回最终的聚集函数结果。
package hive.udaf;
import org.apache.hadoop.hive.ql.exec.UDAF;
import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
public class Avg extends UDAF {
public static class AvgState {
private long mCount;
private double mSum;
}
public static class AvgEvaluator implements UDAFEvaluator {
AvgState state;
public AvgEvaluator() {
super();
state = new AvgState();
init();
}
/** * init函数类似于构造函数，用于UDAF的初始化 */
public void init() {
state.mSum = 0;
state.mCount = 0;
}
/** * iterate接收传入的参数，并进行内部的轮转。其返回类型为boolean * * @param o * @return */
public boolean iterate(Double o) {
if (o != null) {
state.mSum += o;
state.mCount++;
} return true;
}
/** * terminatePartial无参数，其为iterate函数轮转结束后，返回轮转数据， * terminatePartial类似于hadoop的Combiner * * @return */
public AvgState terminatePartial() {
// combiner
return state.mCount == 0 ? null : state;
}
/** * merge接收terminatePartial的返回结果，进行数据merge操作，其返回类型为boolean * * @param o * @return */
public boolean merge(Double o) {
if (o != null) {
state.mCount += o.mCount;
state.mSum += o.mSum;
}
return true;
}
/** * terminate返回最终的聚集函数结果 * * @return */
public Double terminate() {
return state.mCount == 0 ? null : Double.valueOf(state.mSum / state.mCount);
}
}4.UDTF
(1) 继承org.apache.hadoop.hive.ql.udf.generic.GenericUDTF。
(2)实现initialize, process, close三个方法。
UDTF首先会调用initialize方法，此方法返回UDTF的返回行的信息（返回个数，类型）。初始化完成后，会调用process方法，对传入的参数进行处理，可以通过forword()方法把结果返回。最后close()方法调用，对需要清理的方法进行清理。
下面是我写的一个用来切分”key:value;key:value;”这种字符串，返回结果为key, value两个字段。供参考：
import java.util.ArrayList;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
public class ExplodeMap extends GenericUDTF{
@Override
public void close() throws HiveException {
// TODO Auto-generated method stub
}
@Override
public StructObjectInspector initialize(ObjectInspector[] args)
throws UDFArgumentException {
if (args.length != 1) {
throw new UDFArgumentLengthException("ExplodeMap takes only one argument");
}
if (args[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
throw new UDFArgumentException("ExplodeMap takes string as a parameter");
}
ArrayList fieldNames = new ArrayList();
ArrayList fieldOIs = new ArrayList();
fieldNames.add("col1");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldNames.add("col2");
fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames,fieldOIs);
}
@Override
public void process(Object[] args) throws HiveException {
String input = args[0].toString();
String[] test = input.split(";");
for(int i=0; i