大数据测试：sparkSQL取MySQL数据库中的数据

# coding:utf-8
from pyspark.sql import SparkSession
from pyspark.sql.types import *

import os

if __name__ == '__main__':
    os.environ['JAVA_HOME'] = 'C:\Program Files\Java\jdk1.8.0_211'
    print(os.path)
    spark = SparkSession \
        .builder \
        .appName("Python Spark SQL basic example") \
        .config("spark.some.config.option", "some-value") \
        .getOrCreate()
    url="jdbc:mysql://192.168.8.203:3306/test?user=root&password=root"
    props = {}
    props['driver']="com.mysql.jdbc.Driver"
    df=spark.read.jdbc(url,"video", properties =props)

    # df=spark.sql("select * from video")
    df.show()

或者

from pyspark.sql import SparkSession

if __name__ == '__main__':
    
    spark = SparkSession \
    .builder \
    .appName("Python Spark SQL JDBC") \
    .config("spark.some.config.option", "some-value") \
    .getOrCreate()
    
    prop = {}
    prop['user'] = 'root'
    prop['password'] = 'root' 
    prop['driver'] = "com.mysql.jdbc.Driver"
    prop['url'] = 'jdbc:mysql://192.168.8.203:3306/test?useSSL=false'
    prop["dbtable"]="video"
    df = spark.read.jdbc(
        'jdbc:mysql://192.168.8.203:3306/test?useSSL=false',
        "video",
        predicates=["ip_from = 'UNKNOW'", 'id=1'],
        properties=prop)

    df.show()
    pass