Spark求数据集中同一主键记录中的最新数据

/**
 * 测试数据：
 * 1,001,10,2015-10-29
 * 1,001,8.9,2015-10-28
 * 2,002,5,2015-10-27
 * 2,002,3,2015-10-28
 * 3,003,5.9,2015-11-03
 *
 */
object RemoveMultiRecordTest extends App{
  private val sep = "\t"

  val sc = ContextInit.getSparkContext();
  val filepath = "order/multi_record.txt"
  val multiRecord = sc.textFile(filepath).map(_.split(",",-1)).cache()

  /**
 * 组装key_value键-值对，groupByKey,对同一Key下的列表求最大值
 */
 multiRecord.map(line=>(line(0)+sep+line(1),line(2)+sep+line(3))).groupByKey().map(r=>{
    val dateIter = r._2.toIterator
    var maxRecord = dateIter.next
    var maxDateVal = maxRecord.split(sep)(1)

    while(dateIter.hasNext){
      val tempRecord = dateIter.next()
      val tempDate = tempRecord.split(sep)(1)
      if(DateUtil.isBefore(maxDateVal,tempDate)){
        maxDateVal = tempDate
        maxRecord = tempRecord
      }
    }
    Joiner.on(sep).join(r._1,maxRecord)
  }).foreach(println)

  /**
 * 组装key_value键-值对，groupByKey,通过自定义排序，按日期字段排序，获取key下第一个记录
 */
 implicit val KeyOrdering = new Ordering[String] {
    override def compare(x: String, y: String): Int = {
      val date_x = x.split(sep)(1)
      val date_y = y.split(sep)(1)
      date_y.compareTo(date_x)
    }
  }
  multiRecord.map(line=>(line(0)+sep+line(1),line(2)+sep+line(3))).groupByKey().map(r=>{
    val list = r._2.toArray.sorted(KeyOrdering)
    (r._1,list(0))
  }).foreach(println)

  /**
 * 简洁版
 */
 val b = multiRecord.groupBy(a=>(a(0),a(1))).map(r=>{
    val t = r._2.toArray.maxBy(m=>m(3))
    t.mkString(sep)
  }).foreach(println)
}