电商推荐系统项目总结

项目架构

项目过程中的数据转换

注意：图中RDD后跟的不是元素类型，而是其中元素的含义，如：userId即为用户Id，score和rating均为用户对商品的打分

离线阶段数据转化流程

实时阶段数据转化流程

部分代码

商品推荐优先级计算（实时推荐阶段）

def computeProductScores(
	simProducts:scala.collection.Map[Int,scala.collection.immutable.Map[Int,Doub
	le]],userRecentlyRatings:Array[(Int,Double)],topSimProducts: Array[Int]): 
	Array[(Int,Double)] ={

  //用于保存每一个待选商品和最近评分的每一个商品的权重得分
  val score = scala.collection.mutable.ArrayBuffer[(Int,Double)]()

  //用于保存每一个商品的增强因子数
  val increMap = scala.collection.mutable.HashMap[Int,Int]()

  //用于保存每一个商品的减弱因子数
  val decreMap = scala.collection.mutable.HashMap[Int,Int]()

  for (topSimProduct <- topSimProducts; userRecentlyRating <- userRecentlyRatings){
    val simScore = getProductsSimScore(simProducts,userRecentlyRating._1,topSimProduct)
    if(simScore > 0.6){
      score += ((topSimProduct, simScore * userRecentlyRating._2 ))
      if(userRecentlyRating._2 > 3){
        increMap(topSimProduct) = increMap.getOrDefault(topSimProduct,0) + 1
      }else{
        decreMap(topSimProduct) = decreMap.getOrDefault(topSimProduct,0) + 1
      }
    }
  }

  score.groupBy(_._1).map{case (productId,sims) =>
    (productId,sims.map(_._2).sum / sims.length + log(increMap.getOrDefault(productId, 1)) - log(decreMap.getOrDefault(productId, 1)))
  }.toArray.sortWith(_._2>_._2)

}

自定义拦截器

package flume.interceptor;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;

public class LogETLInterceptor implements Interceptor {

    @Override
    public void initialize() {

    }

    @Override
    public Event intercept(Event event) {

        // 1 获取数据
        byte[] body = event.getBody();
        String log = new String(body, Charset.forName("UTF-8"));

        if (log.contains("PRODUCT_RATING_PREFIX:")) {
       		newBody = log.split(":")[1].trim()
	        event.setBody(newBody.getBytes())
	        return event;   
        }else {
        	return null;
        }
    }

    @Override
    public List<Event> intercept(List<Event> events) {

        ArrayList<Event> interceptors = new ArrayList<>();

        for (Event event : events) {
            Event intercept1 = intercept(event);

            if (intercept1 != null){
                interceptors.add(intercept1);
            }
        }

        return interceptors;
    }

    @Override
    public void close() {

    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public Interceptor build() {
            return new LogETLInterceptor();
        }

        @Override
        public void configure(Context context) {

        }
    }
}