基于springboot+vue+爬虫实现电影推荐系统
项目编号：BS-XX-136
一，项目简介
这是一个前后端分离的电影管理和推荐系统，采用Vue.js + Spring Boot技术栈开发，电影数据来源于豆瓣，采用Python爬虫进行爬取相关电影的数据，将数据插入MYSQL数据库，然后在前端进行数据展示。后台主要进行电影相关基本数据的管理功能。给用户推荐的电影数据写入到REDIS数据库中进行存储。推荐算法采用协同过滤算法，采用于ItemCF和UserCF相结合的方式来进行推荐。
二，环境介绍
语言环境：Java: jdk1.8
数据库：Mysql: mysql5.7+Redis
应用服务器：Tomcat: tomcat8.5.31
开发工具：IDEA或eclipse
前端技术栈： - Vue.js - ElementUI - axios
后端技术栈： - Spring Boot - MyBatis - Apache Shiro - Spring Data Redis
三，系统展示
前端分类列表
详情展示
全文检索
电影推荐
后台管理
电影管理
注册
登陆
四，核心代码展示
package com.fivesix.fivesixserver.controller;
import com.fivesix.fivesixserver.entity.Menu;
import com.fivesix.fivesixserver.service.MenuService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@RestController
public class MenuController {
@Autowired
MenuService menuService;
@GetMapping("/api/menu")
public Listpackage com.fivesix.fivesixserver.controller;
import com.fivesix.fivesixserver.entity.Movie;
import com.fivesix.fivesixserver.service.MovieService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
@RestController
public class MovieController {
@Autowired
MovieService movieService;
/*
前台请求接口
*/
@GetMapping("/api/movies")
public List list() throws Exception {
System.out.println("load all movies sorted by rate successfully.");
return movieService.list().stream().sorted(Comparator.comparingDouble(Movie::getRate).reversed()).collect(Collectors.toList());
}
@GetMapping("/api/movies/page/{pageIndex}")
public List listByPageIndex(@PathVariable("pageIndex") int pageIndex) {
System.out.printf("请求起始为%d:的21部电影\n",pageIndex);
return movieService.listByPageIndex(pageIndex).stream().sorted(Comparator.comparingDouble(Movie::getRate).reversed()).collect(Collectors.toList());
}
@GetMapping("/api/movies/category/{cid}/{dateOrRate}")
public Object listByCategory(@PathVariable("cid") int cid, @PathVariable("dateOrRate") int dateOrRate) throws Exception {
List res;
if (cid == 0) {
res = movieService.list();
}else{
res = movieService.listByCategory(cid);
}
if (dateOrRate == 1) return res.stream().sorted(Comparator.comparingDouble(Movie::getRate).reversed()).collect(Collectors.toList());
else return res.stream().sorted(Comparator.comparing(Movie::getDate).reversed()).collect(Collectors.toList());
}
@GetMapping("/api/search")
public List listByKeywords(@RequestParam("keywords") String keywords) {
if (!keywords.equals("")){
System.out.println("search result returned.");
return movieService.listByKeywords(keywords).stream().sorted(Comparator.comparing(Movie::getDate).reversed()).collect(Collectors.toList());
}else{
return null;
}
}
/*
以下为后台请求接口
*/
@PostMapping("/api/admin/content/movie/update")
public Movie save(@RequestBody Movie movie, @RequestParam(value = "changeCategories") String categoriesIsChanged) throws Exception {
Movie movie1 = movieService.getByMovieName(movie.getTitle());
if (movie1 != null) {
if (categoriesIsChanged.equals("true")) {
movieService.updateMovieAndCategories(movie);
System.out.println("update movie and categories.");
} else {
movieService.update(movie);
System.out.println("update movie.");
}
} else {
movieService.save(movie);
System.out.println("add new movie.");
}
return movie;
}
@PostMapping("/api/admin/content/movie/delete")
public void delete(@RequestBody Movie movie) throws Exception{
movieService.deleteById(movie.getId());
System.out.println("delete movie by id successfully.");
}
/*
电影封面上传
*/
@PostMapping("/api/admin/content/movie/cover")
public String coversUpload(MultipartFile file) throws Exception {
String folder = "D:/workspace/fivesix/img/full";
File imageFolder = new File(folder);
//对文件重命名，保留文件的格式png/jpg
String newName = UUID.randomUUID().toString();
File f = new File(imageFolder, newName + file.getOriginalFilename()
.substring(file.getOriginalFilename().length() - 4));
if (!f.getParentFile().exists())
f.getParentFile().mkdirs();
try {
file.transferTo(f);
String imgURL = "http://localhost:8443/api/file/" + f.getName();
return imgURL;
} catch (IOException e) {
e.printStackTrace();
return "";
}
}
}
package com.fivesix.fivesixserver.controller;
import com.fivesix.fivesixserver.entity.User;
import com.fivesix.fivesixserver.result.Result;
import com.fivesix.fivesixserver.service.UserService;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.crypto.SecureRandomNumberGenerator;
import org.apache.shiro.crypto.hash.SimpleHash;
import org.apache.shiro.subject.Subject;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.util.HtmlUtils;
import java.util.List;
@RestController
public class UserController {
private final UserService userService;
public UserController (UserService userService) {
this.userService = userService;
}
@PostMapping(value = "/api/login")
public Result login(@RequestBody User requestUser,@RequestParam(value = "rememberMe") boolean rememberMe) {
String requestUserName = HtmlUtils.htmlEscape(requestUser.getUsername());
Subject subject = SecurityUtils.getSubject();
UsernamePasswordToken usernamePasswordToken = new UsernamePasswordToken(requestUserName,requestUser.getPassword());
usernamePasswordToken.setRememberMe(rememberMe);
try{
subject.login(usernamePasswordToken);
return new Result(200,"login successfully");
} catch (AuthenticationException e) {
e.printStackTrace();
return new Result(400,"账号或密码错误");
}
}
@PostMapping("/api/register")
public Result register(@RequestBody User user) {
String username = HtmlUtils.htmlEscape(user.getUsername());
user.setUsername(username);
//生成盐
String salt = new SecureRandomNumberGenerator().nextBytes().toString();
user.setSalt(salt);
//设置迭代次数
int times = 2;
//生成加密的密码
String encodedPassword = new SimpleHash("md5",user.getPassword(),salt,times).toString();
user.setPassword(encodedPassword);
try {
userService.register(user);
return new Result(200,"register successfully.");
}catch (Exception e) {
e.printStackTrace();
return new Result(400,e.getMessage());
}
}
@GetMapping("api/logout")
public Result logout() {
Subject subject = SecurityUtils.getSubject();
subject.logout();
return new Result(200,"登出成功");
}
@GetMapping("api/authentication")
public Result authenticate() {
return new Result(200,"认证成功");
}
@GetMapping("api/admin/user")
public List getAllUsers() {
return userService.listAll();
}
@PostMapping("api/admin/user/delete")
public Result delete(@RequestBody User user) {
try {
userService.deleteUser(user);
return new Result(200,"删除用户成功");
}catch (Exception e) {
return new Result(400,"删除用户失败");
}
}
@PutMapping("api/admin/user/update")
public Result update(@RequestBody User user) {
try{
userService.updateUser(user);
return new Result(200,"更新用户成功");
}catch (Exception e){
return new Result(400,"更新用户失败");
}
}
}
使用Python来进行电影数据爬取的核心代码
# -*- coding: utf-8 -*-
import scrapy
import json
import re
import time
from douban.items import DoubanItem
from fake_useragent import UserAgent
import random
class MovieHotSpider(scrapy.Spider):
#爬虫的名称，在命令行可以方便的运行爬虫
name = "movie_hot"
allowed_domains = ["movie.douban.com"]
#pro = ['139.224.37.83','115.223.7.110','221.122.91.75']
# 拼接豆瓣电影URL
BASE_URL = 'https://movie.douban.com/j/search_subjects?type=movie&tag=%s&sort=recommend&page_limit=%s&page_start=%s'
MOVIE_TAG = '华语'
PAGE_LIMIT = 20
page_start = 0
domains = BASE_URL % (MOVIE_TAG, PAGE_LIMIT, page_start)
#伪装浏览器
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36"
#,"Cookie":'_vwo_uuid_v2=D65EBF690D9454DE4C13354E37DC5B9AA|3bb7e6e65f20e31141b871b4fea88dc2; __yadk_uid=QBp8bLKHjCn5zS2J5r8xV7327R0wnqkU; douban-fav-remind=1; gr_user_id=0a41d8d1-fe39-4619-827a-17961cf31795; viewed="35013197_10769749_23008813_26282806_34912177_22139960_35003794_30249691_26616244_27035127"; push_noty_num=0; push_doumail_num=0; __utmv=30149280.21320; bid=gplG4aEN4Xc; ll="108288"; ap_v=0,6.0; __utma=30149280.819011260.1572087992.1604448803.1604453561.105; __utmc=30149280; __utmz=30149280.1604453561.105.65.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __gads=ID=eddb65558a1da756-223ab4f88bc400c8:T=1604453562:RT=1604453562:S=ALNI_MZGB_I69qmiL2tt3lm57JVX1i4r2w; __utmb=30149280.4.10.1604453561; dbcl2="213202515:Ip9mjwUAab4"; ck=wxUS; __utma=223695111.897479705.1572088003.1604448803.1604455298.71; __utmb=223695111.0.10.1604455298; __utmc=223695111; __utmz=223695111.1604455298.71.42.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1604455298%2C%22https%3A%2F%2Faccounts.douban.com%2F%22%5D; _pk_ses.100001.4cf6=*; _pk_id.100001.4cf6=e11874c5506d4ab1.1572088003.71.1604455342.1604450364.'
}
#总共爬取的页数
pages = 100
# 爬虫从此开始
def start_requests(self):
print('~~~~爬取列表: '+ self.domains)
yield scrapy.Request(
url = self.domains,
headers=self.headers,
callback=self.request_movies
)
# 分析列表页
def request_movies(self, response):
infos = response.text
# 使用JSON模块解析响应结果
infos = json.loads(infos)
# 迭代影片信息列表
for movie_info in infos['subjects']:
print('~~~爬取电影: ' + movie_info['title'] + '/'+ movie_info['rate'])
# 提取影片页面url，构造Request发送请求，并将item通过meta参数传递给影片页面解析函数
yield scrapy.Request(
url = str(movie_info['url']),
headers = self.headers,
callback = self.request_movie,
dont_filter=True
)
#如果已经爬完pages或者当前标签下没有更多电影时退出
if self.pages > 0 and len(infos['subjects']) == self.PAGE_LIMIT:
self.pages -= 1
self.page_start += self.PAGE_LIMIT
url = self.BASE_URL % (self.MOVIE_TAG,self.PAGE_LIMIT,self.page_start)
time.sleep(5)
print('-----爬取列表: ' + url)
yield scrapy.Request(
url=url,
headers=self.headers,
callback=self.request_movies,
dont_filter=True
)
# 分析详情页
def request_movie(self, response):
#组装数据
movie_item = DoubanItem()
title = response.css('div#content>h1>span:nth-child(1)::text').extract_first()
t = re.findall('[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b\u4e00-\u9fa5_0-9]', title)
#获取非info区域数据
movie_item['title'] = ''.join(t)
movie_item['date'] = response.css('div#content>h1>span.year::text').extract_first()[1:-1]
movie_item['rate'] = response.css('strong.rating_num::text').extract_first()
#movie_item['commentCount'] = response.css('div.rating_sum>a.rating_people>span::text').extract_first()
#movie_item['start'] = '/'.join(response.css('span.rating_per::text').extract())
#movie_item['better'] = '/'.join(response.css('div.rating_betterthan>a::text').extract())
movie_item['abs'] = response.css('#link-report>span::text').extract_first().strip()
movie_item['cover'] = response.css('#mainpic>a>img::attr(src)').extract_first()
# 获取整个信息字符串
info = response.css('div.subject div#info').xpath('string(.)').extract_first()
# 提取所以字段名
fields = [s.strip().replace(':', '') for s in response.css('div#info span.pl::text').extract()]
# 提取所有字段的值
values = [re.sub('\s+', '', s.strip()) for s in re.split('\s*(?:%s):\s*' % '|'.join(fields), info)][1:]
# 处理列名称
for i in range(len(fields)):
if '导演' == fields[i]:
fields[i] = 'director'
if '编剧' == fields[i]:
fields[i] = 'scriptwriter'
if '主演' == fields[i]:
fields[i] = 'actors'
if '类型' == fields[i]:
fields[i] = 'categories'
if '制片国家/地区' == fields[i]:
fields[i] = 'district'
if '语言' == fields[i]:
fields[i] = 'language'
if '片长' == fields[i]:
fields[i] = 'duration'
# 将所有信息填入item
other_info = list(zip(fields,values))
for field,value in other_info:
if field in ['IMDb链接','上映日期','官方网站','又名']:
other_info.remove((field,value))
final_info = dict(other_info[:-1])
movie_item.update(final_info)
# 处理缺失字段
if not 'director' in movie_item.keys():
movie_item['director'] = '/'
if not 'scriptwriter' in movie_item.keys():
movie_item['scriptwriter'] = '/'
if not 'actors' in movie_item.keys():
movie_item['actors'] = '/'
if not 'categories' in movie_item.keys():
movie_item['categories'] = '/'
if not 'district' in movie_item.keys():
movie_item['district'] = '/'
if not 'language' in movie_item.keys():
movie_item['language'] = '/'
if not 'duration' in movie_item.keys():
movie_item['duration'] = '/'
print('~完成爬取电影: ' + movie_item['title'] + '/' + movie_item['rate'])
#将数据加入到字典中
yield movie_item
五，项目总结
项目采用前后端分离的方式来进行开发实现，并进行了数据的爬取操作，相关亮点比较多，业务功能相对较少。主要实现数据的爬取及展示以及相关的推荐功能。