Spark图计算

Spark GraphFrame 介绍：
《图解大数据 | Spark GraphFrames-基于图的数据分析挖掘》
《基于Spark Graphframes的社交关系图谱项目实战》

Spark GraphFrame jar包下载地址：https://spark-packages.org/package/graphframes/graphframes

Spark GraphFrame 官网：https://graphframes.github.io/graphframes/docs/_site/index.html

配置依赖

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0modelVersion>

    <groupId>com.deloittegroupId>
    <artifactId>spark_graphxartifactId>
    <version>1.0-SNAPSHOTversion>

    <properties>
        <maven.compiler.source>8maven.compiler.source>
        <maven.compiler.target>8maven.compiler.target>
    properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.sparkgroupId>
            <artifactId>spark-graphx_2.12artifactId>
            <version>3.1.0version>
        dependency>
        <dependency>
            <groupId>org.apache.sparkgroupId>
            <artifactId>spark-sql_2.12artifactId>
            <version>3.1.1version>
        dependency>
        <dependency>
            <groupId>graphframesgroupId>
            <artifactId>graphframes-spark3.1_2.12artifactId>
            <version>0.8.2version>
            <scope>systemscope>
            <systemPath>${project.basedir}/lib/graphframes-0.8.2-spark3.1-s_2.12.jarsystemPath>
        dependency>
    dependencies>

    <build>
        <resources>
            <resource>
                <directory>libdirectory>
                <targetPath>/BOOT-INF/lib/targetPath>
                <includes>
                    <include>**/*.jarinclude>
                includes>
            resource>
        resources>
    build>
project>

演示代码1：

import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.functions.{col, lit, when}
import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession}
import org.graphframes.GraphFrame

/**
 * 基于社交关系推荐好友
 * A->B and B -> C and A >< C，即A与B双向关系、B与C双向关系，但是A->C没有关系，输出（A，C）
 */
object GraphDataFrame2 {
  def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)

    val spark: SparkSession = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .master("local[*]")
      .getOrCreate()

    val vertices: DataFrame = spark.createDataFrame(Seq(
      ("a", "Alice", 34),
      ("b", "Bob", 36),
      ("c", "Charlie", 37),
      ("d", "David", 29),
      ("e", "Esther", 32),
      ("f", "Fanny", 38),
      ("g", "Gabby", 60)
    )).toDF("id", "name", "age")

    /**
     * friends表示朋友关系；follow表示跟随关系<可看作有相同爱好>
     * Alice和Charies是朋友关系；Esther和Charies没关系
     * Esther是Alice的跟随者，最终推荐Charies给Esther
     */
    val edges: DataFrame = spark.createDataFrame(Seq(
      ("a", "b", "follow"),
      ("a", "c", "friend"),
      ("a", "g", "friend"),
      ("b", "c", "friend"),
      ("c", "a", "friend"),
      ("c", "b", "friend"),
      ("c", "d", "follow"),
      ("c", "g", "friend"),
      ("d", "a", "follow"),
      ("d", "g", "friend"),
      ("e", "a", "follow"),
      ("e", "d", "follow"),
      ("f", "b", "follow"),
      ("f", "c", "follow"),
      ("f", "d", "follow"),
      ("g", "a", "friend"),
      ("g", "c", "friend"),
      ("g", "d", "friend")
    )).toDF("src", "dst", "relationship")

    val graph: GraphFrame = GraphFrame(vertices, edges)


    //计算关联好友
    val relationG: DataFrame = graph.find("(a)-[ab]->(b)")
      .dropDuplicates()
      .selectExpr("a.name as user", "b.name as recommended_user")

    //计算推荐好友
    val recommend: DataFrame = graph.find("(a)-[ab]->(b);(b)-[bc]->(c)")
      .filter("a.id != c.id")
      .filter("ab.relationship = 'follow' and bc.relationship = 'friend'")
      .dropDuplicates()
      .selectExpr("a.name as user", "c.name as recommended_user")
      .except(relationG)

//    recommend.show()
    //获取社交关系链中"[relationship=='friends']>=2" 的数量
    val chain: DataFrame = graph.find("(a)-[ab]->(b);(b)-[bc]->(c);(c)-[cd]->(d)")

    def sumFriends(cnt: Column, relationship: Column): Column = {
      when(relationship === "friend", cnt + 1).otherwise(cnt)
    }

    val condition: Column = {
      Seq("ab", "bc", "cd").foldLeft(lit(0))((cnt, e) => sumFriends(cnt, col(e)("relationship")))
    }
    val chainWithFriends: Dataset[Row] = chain
      .where(condition >= 2)
      .dropDuplicates()
    chainWithFriends.show()

    //过滤操作
    graph
      .filterVertices("age > 30")
      .filterEdges("relationship = 'friend'")
      .dropIsolatedVertices()
      .vertices.show()

    //BFS 广度优先算法 Search from "Alice" for users of age < 32.
    //bfs(fromExpr, toExpr, edgeFilter=None, maxPathLength=10)
    graph.bfs.fromExpr("name = 'Alice'").toExpr("age < 32").run().show()

    graph.bfs.fromExpr("name = 'Alice'").toExpr("age < 32")
      .edgeFilter("relationship = 'friend'")
      .maxPathLength(3).run().show()
    graph.connectedComponents.setAlgorithm("graphx").run().show()

    graph.labelPropagation.maxIter(5).run().show()

    // pageRank pageRank(resetProbability=0.15, sourceId=None, maxIter=None, tol=None)
    // 参数 resetProbability 表示算法里的常数 alpha，默认 0.15；sourceId 指顶点 ID，用于个性化 PageRank 算法，该参数可选；maxIter 指迭代的最大次数；tol 指最终收敛的公差值。
    val pgresult1: GraphFrame = graph.pageRank.resetProbability(0.15).tol(0.01).run()
    pgresult1.vertices.select("id","pagerank").show()
    pgresult1.edges.select("src","dst","weight").show()

    val pgresult2: GraphFrame = graph.pageRank.resetProbability(0.15).maxIter(10).sourceId("a").run()
    val pgresult3: GraphFrame = graph.parallelPersonalizedPageRank.resetProbability(0.15).maxIter(10).sourceIds(Array("a","b","c")).run()

    //最短路径算法 shortestPaths(landmarks) 但返回结果只有距离值，并不会返回完整的路径
    val results: DataFrame = graph.shortestPaths.landmarks(Seq("a", "d")).run()
    results.show()
    results.select("id","distances").show()

    graph.triangleCount.run().show()

  }
}

演示代码2

import org.apache.log4j.{Level, Logger}
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
import org.graphframes.GraphFrame

/**
 * oneid 合并
 * 数据满足规则相同，则进行合并
 */
object GraphDataFrame3 {
  def main(args: Array[String]): Unit = {
    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)

    val spark: SparkSession = SparkSession
      .builder()
      .appName(this.getClass.getName)
      .master("local[*]")
      .getOrCreate()

    val vertices: DataFrame = spark.createDataFrame(Seq(
      (1L, "Alice", 34, 177311),
      (2L, "Bob", 36, 177312),
      (3L, "Charlie", 34,177311),
      (4L, "Charlie", 29,177311),
      (5L, "Esther", 36,177312),
      (6L, "Fanny", 38,138211),
      (8L, "Charlie", 29,177311),
      (7L, "Gabby", 60,139211)
    )).toDF("id", "name", "age","phone")

    val nameEdges: DataFrame = vertices.join(vertices, Seq("name", "phone"), "inner")
      .toDF("name", "phone", "srcId", "srcAge", "dstId", "dstAge")
      .filter("srcId != dstId")
      .selectExpr("srcId as src", "dstId as dst", "'name+phone' as relationship")

    val ageEdges: DataFrame = vertices.join(vertices, Seq("age", "phone"), "inner")
      .toDF("age", "phone", "srcId", "srcName", "dstId", "dstName")
      .filter("srcId != dstId")
      .selectExpr("srcId as src", "dstId as dst", "'age+phone' as relationship")

    val edges: Dataset[Row] = nameEdges.union(ageEdges)

    edges.show()
    val graph: GraphFrame = GraphFrame(vertices, edges)

    graph.triplets.show()

    val components: DataFrame = graph.connectedComponents.setAlgorithm("graphx").run()
    components.orderBy("component","id").show()

  }
}