Spark udf,udaf

udf

def filterTdWithOp(operator: String): Boolean = {
    val x = ".*中国联通.*"
    val y = ".*CHN-UNICOM.*"
    val z = ".*China Unicom.*"
    if (operator!=null && operator.matches(x + "|" + y + "|" + z))
      true
    else
      false
  }
sqlContext.udf.register("filterTdWithOp", filterTdWithOp _)
td.registerTempTable("td")
val tdOp = sqlContext.sql("select *,filterTdWithOp(operator) as fiOp from td").filter("fiOp = true")

udaf

去看官网比较好

object  MaxPoint extends Aggregator[Input,Input, Input] {
override def zero: Input = Input(0,0,0,"")
  // Combine two values to produce a new value. For performance, the function may modify `buffer`
  // and return it instead of constructing a new object
  override def reduce(b: Input, a: Input): Input = {
    if(b.pointNum>a.pointNum)
      b
    else
      a
  }
  // Merge two intermediate values
  override def merge(b: Input, a: Input): Input = {
    if(b.pointNum>a.pointNum)
      b
    else
      a
  }

  override def finish(reduction: Input): Input = reduction
}

spark2.0之后引入了两个新方法