04 graphx 从源节点到其他节点的路径 scala & java 版本

前言

呵呵 最近刚好有一些需要使用到 图的相关计算 

然后 需求是 需要计算图中 源点 到 目标节点 的所有路径 

另外本文会提供一个 scala 版本的测试用例, 以及 一个 java 版本的测试用例(写的有点惨) 

 

环境如下 : spark2.4.5 + scala2.11 + jdk8

java 版本的代码基于 : spark-graphx_2.12 2.4.6 

另外 java 版本的代码, 会遇到各种问题, 这里也会 介绍一下(写出来也是很不美观, 可读性也不太好) 

本文的图的样例 基于 03 graphx 从 SSSP 来看 pregel 的图

 

 

测试用例

package com.hx.test01

import org.apache.spark.graphx.{Edge, EdgeDirection, Graph, VertexId}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.util.control.Breaks._

/**
  * Test4AllPaths
  *
  * @author Jerry.X.He <[email protected]>
  * @version 1.0
  * @date 2020/6/18 20:53
  */
object Test4AllPaths {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setMaster("local").setAppName("Test02SSSP")
    val sc = new SparkContext(conf)
    val sourceId: VertexId = 0 // The ultimate source

    // 创造一个边的RDD, 包含各种关系
    val edges: RDD[Edge[Double]] = sc.parallelize(Array(
      Edge(3L, 7L, 1.0d),
      Edge(5L, 3L, 1.0d),
      Edge(2L, 5L, 1.0d),
      Edge(5L, 7L, 1.0d),
      Edge(0L, 3L, 1.0d),
      Edge(3L, 2L, 1.0d),
      Edge(7L, 9L, 1.0d),
      Edge(0L, 5L, 1.0d)
    ))

    // 创造一个点的 RDD
    val vertexes: RDD[(VertexId, (Double, Seq[Seq[VertexId]]))] = edges
      .flatMap(edge => Array(edge.srcId, edge.dstId))
      .distinct()
      .map(id =>
        if (id == sourceId) (id, (0, Seq[Seq[VertexId]](Seq[VertexId](sourceId))))
        else (id, (Double.PositiveInfinity, Seq[Seq[VertexId]]()))
      )
    val defaultVertex = (-1.0d, Seq[Seq[VertexId]]())

    // A graph with edge attributes containing distances
    val initialGraph: Graph[(Double, Seq[Seq[VertexId]]), Double] = Graph(vertexes, edges, defaultVertex)

    println(" edges as follow : ")
    initialGraph.edges.foreach(println)

    val sssp = initialGraph.pregel((Double.PositiveInfinity, Seq[Seq[VertexId]]()), Int.MaxValue, EdgeDirection.Out)(
      // Vertex Program
      (id, dist, newDist) => (Math.min(dist._1, newDist._1), dist._2.union(newDist._2).distinct),
      // Send Message
      triplet => {
        // if is `Double.PositiveInfinity`, return
        if (triplet.srcAttr._1 > Double.MaxValue - 1) {
          Iterator.empty
          // if is edge to self, return
        } else if (triplet.srcId == triplet.dstId) {
          Iterator.empty
        } else {
          val paths = triplet.srcAttr._2
          val isSourceNode = paths.forall(_.size == 1)
          val allExists = paths.forall(_.contains(triplet.dstId))
          val containCircle = paths.foldLeft(false)((result, path) => result || (path.contains(triplet.srcId) && path.contains(triplet.dstId)))
          // 如果不是源节点, 并且 存在cycle 或者 目标节点在所有路径都存在, return
          if (!isSourceNode && (allExists || containCircle)) {
            //        if(triplet.srcAttr._1 > 100) {
            Iterator.empty
            // 当前所有路径 + 目标节点, 传递消息给 目标节点
          } else {
            var newSeqList = Seq[Seq[VertexId]]()
            triplet.srcAttr._2.foreach(seq => {
              if (!(seq.contains(triplet.srcId) && seq.contains(triplet.dstId))) {
                newSeqList = newSeqList :+ (if (seq.contains(triplet.dstId)) seq else seq :+ triplet.dstId)
              }
            })
            Iterator((triplet.dstId, (triplet.srcAttr._1 + triplet.attr, newSeqList)))
          }
        }
      },
      //Merge Message
      (a, b) => (Math.min(a._1, b._1), (a._2.union(b._2).distinct)))

    println(sssp.vertices.collect().foreach(entry => {
      println(s" vertex : ${entry._1}, sssp : ${entry._2._1}, pathLength : ${entry._2._2.size}, paths : ${entry._2._2} ")
    }))

    printMaxPathLength(sssp)

  }

  /**
    * 打印到各个节点路径中, 最长的路径对应的节点 以及 路径长度
    *
    * @param sssp sssp
    * @return void
    * @author Jerry.X.He
    * @date 2020/6/19 21:26
    * @since 1.0
    */
  def printMaxPathLength(sssp: Graph[(Double, Seq[Seq[VertexId]]), Double]): Unit = {
    var maxLength = 0
    var pair: (VertexId, Int) = null
    for (entry <- sssp.vertices.collect()) {
      val pathList = entry._2._2
      breakable {
        if (pathList.isEmpty) {
          break
        }

        val maxPath = pathList.sortBy(_.size).reverse.head
        if (maxPath.size > maxLength) {
          maxLength = maxPath.size
          pair = (entry._1, maxLength)
        }
      }
    }
    println(pair)
  }

}

 

执行结果如下图, 统计了 0 到其他节点的 所有的路径, 以及最短路径长度  

 

 

java 版本

package com.hx.test06;

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.graphx.Edge;
import org.apache.spark.graphx.EdgeDirection;
import org.apache.spark.graphx.EdgeTriplet;
import org.apache.spark.graphx.Graph;
import org.apache.spark.rdd.RDD;
import org.apache.spark.storage.StorageLevel;
import scala.Function1;
import scala.Function2;
import scala.Function3;
import scala.Tuple2;
import scala.collection.Iterator;
import scala.collection.mutable.ArrayStack;
import scala.reflect.ClassTag;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * Test27AllPaths
 *
 * @author Jerry.X.He <[email protected]>
 * @version 1.0
 * @date 2020-06-20 11:29
 */
public class Test27AllPaths {

  // Test27AllPaths
  public static void main(String[] args) {
    SparkConf conf = new SparkConf().setMaster("local").setAppName("Test27AllPaths");
    SparkContext sc = new SparkContext(conf);
//    Graph graph = GraphGenerators.rmatGraph(sc, 10, 20);
    Long sourceId = 0L;

    ArrayStack> edges = new ArrayStack<>();
    edges.push(new Edge(3L, 7L, 1.0d));
    edges.push(new Edge(5L, 3L, 1.0d));
    edges.push(new Edge(2L, 5L, 1.0d));
    edges.push(new Edge(5L, 7L, 1.0d));
    edges.push(new Edge(0L, 3L, 1.0d));
    edges.push(new Edge(3L, 2L, 1.0d));
    edges.push(new Edge(7L, 9L, 1.0d));
    edges.push(new Edge(0L, 5L, 1.0d));

    Set vertexSet = new HashSet<>();
    edges.foreach(edge -> {
      vertexSet.add(edge.srcId());
      vertexSet.add(edge.dstId());
      return null;
    });
    ArrayStack>>>> vertexAndInfoList = new ArrayStack<>();
    for (Long vertexId : vertexSet) {
      Tuple2 vertexInfo = new Tuple2(Double.MAX_VALUE, new ArrayList<>());
      if (vertexId.equals(sourceId)) {
        List> sourcePathList = new ArrayList<>();
        List