python调用远程spark资源

# -*- encoding: UTF-8 -*-
# @auther:hx
# @datetime:2018-03-01
from pyspark import SparkContext
import os
os.environ['HADOOP_HOME']='D:\\system\\hadoop-common-2.2.0-bin-master'
os.environ['PYSPARK_PYTHON'] = '/opt/anaconda34/bin/python'
os.environ['PYSPARK_DRIVER_PYTHON']='/opt/anaconda34/bin/python'

from pyspark import SparkConf
conf = SparkConf().setMaster("spark://192.168.0.130:7077").setAppName("Fisrt") \
        .set("spark.num.executors", "1")\
        .set("spark.executor.cores", "2")\
        .set("spark.executor.memory", "600m")\
        .set("spark.driver.memory", "600m")
sc = SparkContext(conf=conf)

logData = sc.textFile("hdfs://hx:9000/data/a.txt").cache()
numAs = logData.filter(lambda s: 'a' in s).count()
numBs = logData.filter(lambda s: 'b' in s).count()
print("Lines with a: %i, lines with b: %i"%(numAs, numBs))
sc.stop()

#注意：要求windows下的python版本和远程linux下的python版本需要一致

输出结果：

D:\app\anaconda3\python.exe E:/workspacePycharm/wordcount/test.py
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
18/10/24 15:58:57 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Lines with a: 1, lines with b: 1

Process finished with exit code 0