基于spark+hadoop+hive大数据分析的电影推荐系统的设计与实现

作者主页：计算机毕设小程
精彩专栏推荐订阅：在 下方专栏

Java实战项目

一、开发介绍

1.1 开发环境

• 技术栈：spark+hadoop+hive
• 离线ETL+在线数据分析 (OLAP)+流计算+机器学习+图计算

二、系统介绍

2.1图片展示

用户注册登录模块：
首页模块：
电影信息模块：

三、部分代码设计

3.1.部分代码如下：

package bigdata.scala.spark.streaming.task.testscalaapp

import bigdata.java.framework.spark.kafka.{
   SKProcessor, SimpleSparkKafka}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.streaming.api.java.JavaInputDStream

object ScalaDemo {
   
  def main(args: Array[String]): Unit = {
   
    //在idea中调试时，Program arguments 参数
    //appname:FindDataStreaming topics:yxgk.a_Cashchk_Flow duration:2 maxnum:2000
    //appname=应用程序名称
    //topics=消费的kafka主题，多个主题用逗号隔开
    //duration=每隔多少秒执行一个spark批次
    //maxnum=每个批次获取多少数据，每隔批次消费的总数据量等于分区数量*秒数*maxnum
    val simpleSparkKafka = new SimpleSparkKafka(args)
    simpleSparkKafka.execute(new SKProcessor {
   
      override def process(directStream: JavaInputDStream[ConsumerRecord[String, String]]): Unit = {
   
        val stream = directStream.inputDStream
        stream.foreachRDD{
   rdd=>
          rdd.foreachPartition {
    partitionOfRecods =>
            partitionOfRecods.foreach(println)
          }
        }
      }
    }).start()
  }
}

package bigdata.java.spark.streaming.task.datasynchro;

import bigdata.java.framework.spark.util.JsonUtil;
import bigdata.java.framework.spark.util.StrUtil;
import scala.Tuple3;

import java.util.List;

public class DataModel {
   

    /**
     * 表名
     */
    String table;
    /**
     * 主键
     */
    List<Tuple3<String,String,String>> primary_Keys;
    /**
     * 字段
     */
    List<Tuple3<String,String,String>> fields;
    /**
     * 操作类型
     */
    String op_Type;
    /**
     * sql语句
     */
    String sql;

    String json;

    public String getJson() {
   
        return json;
    }

    public void setJson(String json) {
   
        this.json = json;
    }

    public List<Tuple3<String, String, String>> getFields() {
   
        return fields;
    }

    public Tuple3<String, String, String> getPrimaryValue(String key)
    {
   
        for (int i = 0; i < primary_Keys.size(); i++) {
   
            Tuple3<String, String, String> tuple3 = primary_Keys.get(i);
            if(tuple3._1().equals(key))
            {
   
                return tuple3;
            }
        }
        return null;
    }

    public Tuple3<String, String, String> getFieldValue(String key)
    {
   
        for (int i = 0; i < fields.size(); i++) {
   
            Tuple3<String, String, String> tuple3 = fields.get(i);
            if(tuple3._1().equals(key))
            {
   
                return tuple3;
            }
        }
        return null;
    }

    public void setFields(List<Tuple3<String, String, String>> fields) {
   
        this.fields = fields;
    }

    public String getTable() {
   
        return table;
    }

    public void setTable(String table) {
   
        this.table = table;
    }

    public List<Tuple3<String,String,String>> getPrimary_Keys() {
   
        return primary_Keys;
    }

    public void setPrimary_Keys(List<Tuple3<String,String,String>> primary_Keys) {
   
        this.primary_Keys = primary_Keys;
    }

    public String getOp_Type() {
   
        return op_Type;
    }

    public void setOp_Type(String op_Type) {
   
        this.op_Type = op_Type;
    }

    /**类型转换
     * @param typeName
     * @return
     */
    public static String covertType(String typeName)
    {
   
        String type="";
        if(typeName.contains("String"))
        {
   
            type="String";
        }
        else if(typeName.contains("Integer"))
        {
   
            type="Integer";
        }
        else if(typeName.contains("Boolean"))
        {
   
            type="Boolean";
        }
        else if(typeName.contains("Long"))
        {
   
            type="Long";
        }
        else if(typeName.contains("Double"))
        {
   
            type="Double";
        }
        else if(typeName.contains("BigDecimal"))
        {
   
            type="BigDecimal";
        }
        else
        {
   
            type="";
//                            continue;
        }
        return type;
    }

    /**根据类型，生成对于的数据库插入value是否带有引号
     * @param value
     * @param type
     * @return
     */
    public String getTyepAferValue(String value,String type)
    {
   
        String afterValue = "";
        if(type.contains("String"))
        {
   
            afterValue="\""+value+"\"";
        }
        else if(type.contains("Integer"))
        {
   
            afterValue= value;
        }
        else if(type.contains("Boolean"))
        {
   
            afterValue= value;
        }
        else if(type.contains("Long"))
        {
   
            afterValue= value;
        }
        else if(type.contains("Double"))
        {
   
            afterValue= value;
        }
        else if(type.contains("BigDecimal"))
        {
   
            afterValue= value;
        }
        else
        {
   
            afterValue="";
        }
        return afterValue;
    }

    public String getSql() {
   
        return sql;
    }

    /**
     * 生成sql语句
     */
    public void createSql() {
   
        //获取条件，根据主键
        String where ="";
        if(getPrimary_Keys().size() < 1)
        {
   //没有发现主键，那么直接报错
            throw new RuntimeException("no find primary key");
        }
        for (int i = 0