大数据开发大作业3

一级目录

docker cp /tmp/test_data/ml-100k c5f727be11c8:

mkdir 
hdfs dfs -mkdir /b
hdfs dfs -put /ml-100k /b
hdfs dfs -put /ml-100k /a
hdfs dfs -ls /a/ml-100k
hdfs dfs -ls /c/ml-100k

hdfs dfs -ls /dzy6
hdfs dfs -cat /dzy6/part-00003


hdfs dfs -ls /test_data

docker cp /tmp/test_data/djd.jar c5f727be11c8:home/


hdfs dfs -cat /djd1/part-00000
djd1

#执行最终结果
spark-submit --class org.example.homework2 --master yarn  --deploy-mode client /home/gpb.jar hdfs://hbase-master.hadoop-docker:54310/a/ml-100k/u.data  hdfs://hbase-master.hadoop-docker:54310/dzy7/


spark-submit --class $1 --master yarn  --deploy-mode client $2 $3 $4

bash test.sh org.example.homework2 /home/gpb.jar hdfs://hbase-master.hadoop-docker:54310/a/ml-100k/u.data hdfs://hbase-master.hadoop-docker:54310/dzy6/

bash test.sh org.example.homework2 /home/djd.jar hdfs://hbase-master.hadoop-docker:54310/a/ml-100k/u.data hdfs://hbase-master.hadoop-docker:54310/djd1/

bash test.sh org.example.homework2 /home/djd.jar hdfs://hbase-master.hadoop-docker:54310/a/ml-100k/u.data hdfs://hbase-master.hadoop-docker:54310/djd1/

org.example.homework2
/home/gpb.jar
hdfs://hbase-master.hadoop-docker:54310/a/ml-100k/u.data
hdfs://hbase-master.hadoop-docker:54310/dzy6/

spark-submit --class “Recomment” /usr/local/spark/mycode/Kmean/target/scala-2.11/simple-project_2.11-1.0.jar

#查看hdfs 的test_data文件夹的文件
hdfs dfs -ls /dzy3

#查看hdfs 的part-00000文件
hdfs dfs -cat /dzy3/part-00000
hdfs dfs -cat /dzy3/part-00001

链接: SBT命令行打包spark程序

spark-submit --class “org.example.homework2” /home/gpb.jar

docker cp /tmp/test_data/gpb.jar c5f727be11c8:home/