Spark基础：如何遍历dataframe

#coding:utf-8
from pyspark import SparkConf,SparkContext
from pyspark.sql import HiveContext
import datetime
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

# 初始化
conf=SparkConf().setAppName("genUserDescWords").setMaster("local")
sc=SparkContext(conf=conf)
hc=HiveContext(sc)

prt_dt = (datetime.datetime.now()-datetime.timedelta(days=2)).strftime('%Y-%m-%d')
sql = "select * from test.user_likes_info where prt_dt='"+prt_dt+"' limit 100"
# 生成DataFrame
df = hc.sql(sql)
# DataFrame转list
rows=df.collect()
cols=df.columns
cols_len=len(cols)
all_list=[]
for row in rows:
	user_info=[]
	likes_info=[]
	prt_info=[]
	most_like=[]
	for idx,col in enumerate(cols):
		if idx <3:
			user_info.append(row[col])
		elif idx == cols_len:
			prt_info.append(row[col])
		else:
			if idx == 3 and row[col]:
				most_like = row[col]
			else:
				likes_info.append(row[col])
	if most_like and most_like[0] in likes_info:
		likes_info.pop(likes_info.index(most_like[0]))
		likes_info+=[most_like[0]+"，您大多数时间都花在这上面"]
	likes_info=list(set(likes_info))
	all_list.append('\t'.join(user_info+likes_info+prt_info))
# list转化成RDD
rdd=sc.parallelize(all_list)
# 通过RDD将数据保存到HDFS
rdd.saveAsTextFile("hdfs://192.168.2.61:8020/user/hhc/spark_test")

将上述代码保存py文件，然后再通过命令：/usr/bin/spark-submit file_name.py，即可运行。