java 操作ES 增删改查总结
本实例框架采用 nutz+ES，如果 您查找相关知识，请注意过滤不相干的信息
controller 层：
package com.test.ownasset.controller;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Workbook;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.WildcardQueryBuilder;
import org.nutz.dao.Cnd;
import org.nutz.ioc.Ioc;
import org.nutz.ioc.annotation.InjectName;
import org.nutz.ioc.loader.annotation.Inject;
import org.nutz.ioc.loader.annotation.IocBean;
import org.nutz.mvc.annotation.AdaptBy;
import org.nutz.mvc.annotation.At;
import org.nutz.mvc.annotation.Chain;
import org.nutz.mvc.annotation.Fail;
import org.nutz.mvc.annotation.Ok;
import org.nutz.mvc.annotation.Param;
import org.nutz.mvc.upload.TempFile;
import org.nutz.mvc.upload.UploadAdaptor;
import com.test.Cupid;
import com.test.annotation.AuthBy;
import com.test.annotation.Log;
import com.test.commons.mvc.Pager;
import com.test.commons.mvc.validate.annotation.Validations;
import com.test.commons.mvc.view.exhandler.ExceptionWrapper;
import com.test.ownasset.data.AllAddressInProvice;
import com.test.ownasset.data.OwnUsedAsset;
import com.test.ownasset.service.OwnAssetService;
import com.test.ownasset.service.OwnUsedSegmentService;
import com.test.ownasset.service.esService.OwnUsedAssetService;
import com.test.ownasset.util.DateUtil;
import com.test.util.UploadAndDownloadUtil;
@IocBean(name = "ownAssetController")
@AuthBy(login=false)
@At("/ownAsset/ownAsset")
@Fail("error")
public class OwnAssetController {
//ownAsset本身Service 单个IP 处理
@Inject("refer:ownAssetService")
private OwnAssetService ownAssetService;
// 自用资产IP 段
@Inject("refer:ownUsedSegmentService")
private OwnUsedSegmentService ownUsedSegmentService;
/**
* 跳到运营商接口首页
*/
@At("/mainTree")
@Ok("jsp:jsp.ownAsset.main")
@Log(isEnabled = false)
public void viewTree() {
}
/**
* 跳到运营商接口首页
*/
@At("/main")
@Ok("jsp:jsp.ownAsset.ownAsset.list")
@Log(isEnabled = false)
public void viewTab() {
}
/**
* 进入JSP页面--新增iframe
*/
@At("/tab/insert")
@Ok("jsp:jsp.ownAsset.ownAsset.add")
@Log(isEnabled = false)
public void insertTab() {
}
/**
* 新增(ajax)
*/
@At("/insert/ajax")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Fail("json")
// 失败处理
@Chain("validate")
// 校验方法
@Log(isEnabled = false)
public void insertAjax(Ioc ioc, HttpServletRequest request,
HttpSession session, @Param("..") OwnUsedAsset obj) {
OwnUsedAssetService.getInstance().insertAddressToDB(obj);
}
/**
* 进入JSP页面--更新iframe
*/
@At("/tab/update")
@Ok("jsp:jsp.ownAsset.ownAsset.edit")
@Log(isEnabled = false)
public void updateTab() {
}
/**
* 更新(ajax)
*/
@At("/update/ajax")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Chain("validate")
@Validations("validateUpdate")
// 校验
@Fail("json")
// 失败处理
@Log(isEnabled = false)
public void updateAjax(HttpServletRequest request, HttpSession session,
@Param("..") OwnUsedAsset obj, @Param("..") Pager pager) {
OwnUsedAssetService.getInstance().updateInfo(obj);
}
/**
* 加载列表(ajax)
*/
@At("/list/ajax")
@Ok("json")
@Log(isEnabled = false)
public Pager listAjax(Ioc ioc, HttpServletRequest request,
HttpSession session, @Param("..") Pager pager) {
String context = request.getParameter("search");
if(context == null || context.trim().equals("")){
return OwnUsedAssetService.getInstance().getList(pager, null);
}// 模糊查询
else{
try {
BoolQueryBuilder qb = QueryBuilders.boolQuery();
// 组合 模糊查询 should
WildcardQueryBuilder ip = QueryBuilders.wildcardQuery("ip", "*"+context+"*");
WildcardQueryBuilder area = QueryBuilders.wildcardQuery("area", "*"+context+"*");
WildcardQueryBuilder remark = QueryBuilders.wildcardQuery("remark", "*"+context+"*");
qb.should(ip).should(area).should(remark);//
return OwnUsedAssetService.getInstance().getFuzzyList(pager, qb);
} catch (Exception e) {
e.printStackTrace();
return OwnUsedAssetService.getInstance().getList(pager, null);
}
}
}
/**
* 模糊查询(ajax)
*
* @throws UnsupportedEncodingException
*/
@At("/search/ajax")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Log(isEnabled = false)
public Pager searchListAjax(Ioc ioc, HttpServletRequest request,
HttpSession session, @Param("..") Pager pager,
@Param("search") String search) throws UnsupportedEncodingException {
search = java.net.URLDecoder.decode(search, "utf-8");
// 组合 模糊查询 should
BoolQueryBuilder qb = QueryBuilders.boolQuery();
// 组合 模糊查询 should
WildcardQueryBuilder ip = QueryBuilders.wildcardQuery("ip", "*"+search+"*");
WildcardQueryBuilder area = QueryBuilders.wildcardQuery("area", "*"+search+"*");
WildcardQueryBuilder remark = QueryBuilders.wildcardQuery("remark", "*"+search+"*");
qb.should(ip).should(area).should(remark);//
return OwnUsedAssetService.getInstance().getFuzzyList(pager, qb);
}
/**
* 查询单条(ajax)
*/
@At("/detail/ajax")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Log(isEnabled = false)
public OwnUsedAsset readAjax(Ioc ioc, HttpServletRequest request,
HttpSession session, @Param("id") long id) {
Map qbMap = new HashMap();
qbMap.put("id", id);
return OwnUsedAssetService.getInstance().getEntry(qbMap);
}
/**
* 删除(ajax)
*
* @return
*/
@At("/delete/ajax")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Log(isEnabled = false)
public boolean deleteAjax(Ioc ioc, HttpServletRequest request,
HttpSession session, @Param("ids") long[] ids, Cnd cnd) {
boolean flag = true;
try {
OwnUsedAssetService.getInstance().deleteInfo(ids);
} catch (Exception e) {
flag = false;
e.printStackTrace();
}
return flag;
}
@At("/isValidName")
@Ok("json")
@Log(isEnabled = false)
public boolean isValidName(HttpServletRequest request, @Param("filename") String name) {
boolean flag = false;
if (name.toLowerCase().endsWith(".xls")||name.toLowerCase().endsWith(".xlsx")) {
flag = true;
}
return flag;
}
/**
* 新增ip验证唯一性
*/
@At("/getAddCheck")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Log(isEnabled = false)
public boolean getuk(Ioc ioc, HttpServletRequest request,
HttpSession session, @Param("IP") String IP) {
if (IP != null && !IP.equals("")) {
HashMap qbMap = new HashMap();
qbMap.put("ip", IP);
OwnUsedAsset obj = OwnUsedAssetService.getInstance().getEntry(qbMap);
if (obj == null) {
return true;
}
}
return false;
}
/**
* 修改 -->ip验证唯一性
*/
@At("/getUpdateCheck")
// 访问路径与传统方式的路径区分开
@Ok("json")
// 无论成功或是失败，都使用json作为输出
@Log(isEnabled = false)
public boolean getupdateuk(Ioc ioc, HttpServletRequest request,
HttpSession session,@Param("id") String id, @Param("IP") String IP) {
if (IP != null && !IP.equals("") && id != null && id != "") {
Long idTemp=Long.parseLong(id);
BoolQueryBuilder qb = QueryBuilders.boolQuery();
QueryBuilder ipQb = QueryBuilders.termQuery("ip", IP);
QueryBuilder idQb = QueryBuilders.termQuery("id", idTemp);
qb.must(ipQb).mustNot(idQb);
OwnUsedAsset obj = OwnUsedAssetService.getInstance().getEntry(qb);
//不判断本条记录信息.
if (obj == null) {
return true;
}
}
return false;
}
/**
* 导入自用资产信息
*
* @param request
* @param tempFile
*
*/
@At("/import")
@Ok("jsp:jsp.ownAsset.ownAsset.list")
@Fail("error")
@AdaptBy(type = UploadAdaptor.class, args = {})
@Log(value = "导入自用资产Excel入库")
public void importExcel(HttpServletRequest request,
@Param("filename") TempFile tempFile ){
if (tempFile != null) {
File file = tempFile.getFile();
try {
String msg = ownAssetService.importExcel2(file);
ownUsedSegmentService.importExcel(file);
request.setAttribute("successFlag", msg);
} catch (Exception e) {
request.setAttribute("successFlag", false);
}
}
}
@At("/importTemplate")
@Fail("error")
@Ok("void")
@Log(value = "自用地址段附件下载")
public void downloadFile(HttpServletRequest request,
ServletContext context,
HttpServletResponse response)throws FileNotFoundException, IOException{
String path = Cupid.getContextValueAs(String.class,"web.dir");
path = path+File.separator+"downloads"+File.separator+"ownAsset"+File.separator+"自用地址段导入样例.xlsx";
File tempFile = new File(path);
//检查文件是否存在，若不存在则抛出错误提示
if (tempFile.isFile() && tempFile.exists()) {
UploadAndDownloadUtil.downLoad(path,response, request, false);
}else{
throw ExceptionWrapper.wrapError(new ExceptionWrapper.EC("affixal.download.noFile", false));
}
}
@At("/export")
@Fail("error")
@Ok("void")
@Log(value = "导出自用地址段表格")
public void exportExcel(@Param("ids") long[] ids,HttpServletRequest request,
HttpServletResponse response,Ioc ioc,HttpSession session, @Param("..") Pager pager,
@Param("search") String search){
try {
OutputStream os = null;
Workbook wb = ownAssetService.writeExcel(ids,ioc, session, pager, search,0);
String timeStr = DateUtil.getNowTimeStr().replaceAll(" ", "-").replaceAll(":", "-").replaceAll("-", "").substring(0,12);//201704271032 时间标志
String fileName = "自用地址段导出信息"+timeStr+".xlsx";
response.setContentType("application/vnd.ms-excel");
response.setHeader("Content-disposition", "attachment;filename="+ URLEncoder.encode(fileName, "utf-8"));
os = response.getOutputStream();
wb.write(os);
os.close();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
第一层 service：
package com.test.ownasset.service;
import java.io.File;
import java.io.FileInputStream;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import javax.servlet.http.HttpSession;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.WildcardQueryBuilder;
import org.nutz.dao.Cnd;
import org.nutz.dao.Condition;
import org.nutz.ioc.Ioc;
import org.nutz.ioc.loader.annotation.IocBean;
import org.nutz.mvc.annotation.Param;
import com.test.commons.crud.service.AbstractTableTemplateService;
import com.test.commons.mvc.Pager;
import com.test.ownasset.cache.OwnAssetConfig;
import com.test.ownasset.data.OwnUsedAsset;
import com.test.ownasset.service.esService.AllAddressInProviceService;
import com.test.ownasset.service.esService.OwnUsedAssetService;
import com.test.ownasset.task.service.EsFlowService;
import com.test.ownasset.util.ExcelUtil;
import com.test.ownasset.util.IpUtil;
/**
*
* @author zhangqingzhou
*/
@IocBean(name = "ownAssetService")
public class OwnAssetService {
private final Properties properties = OwnAssetConfig.getInstance().getProperties();
/**
* 是否是 系统初始化 导入数据
*/
private boolean isInit = Long.valueOf(properties.getProperty("ownAssetInitImportData", "1")) == 1?true :false;
/**
* 导入EXCEL
* 分段导入,避免 内存溢出
* @param file 路由端口信息文件
* @throws Exception 导入异常
*/
public String importExcel2(File file) throws Exception {
String importInfo ="true";
if (file.isFile()) {
String fileName = file.getName().toLowerCase();
if (fileName.endsWith(".xls") || fileName.endsWith(".xlsx")) {
getExcelAnImport(file);
}
}
return importInfo;
}
/**
* 分段导入
* @param file
* @return
*/
public void getExcelAnImport(File file) {
HashMap excelMap = new HashMap();
// 记录首次出现异常的 行数
boolean isInsert = isInit;
int line = 1;
//Excel相关变量
try {
Workbook workbook = WorkbookFactory.create(new FileInputStream(file));
int sheetNum = workbook.getNumberOfSheets();
List list = new ArrayList<>();
if (sheetNum > 0) {
Sheet sheet;
int count;
Row row;
sheet = workbook.getSheetAt(0);
count = sheet.getLastRowNum(); // 统计表格 的 行数
for (int i = 1; i < count + 1; i++) {//循环行
line++;
row = sheet.getRow(i); // 获取索引对应行的 信息
// 子网IP
String subnetIP = ExcelUtil.getCellValue(row.getCell(0)).trim();
// 子网掩码
int subnetMask = Integer.parseInt( ExcelUtil.getCellValue(row.getCell(1)));
// 自用资产区域(地级市)
String area = ExcelUtil.getCellValue(row.getCell(3)).trim();
// 备注信息
String remark = ExcelUtil.getCellValue(row.getCell(4))+";"+ExcelUtil.getCellValue(row.getCell(5))+";"+ExcelUtil.getCellValue(row.getCell(6));
// 获取起始IP + 终止IP
String[] ipArr = IpUtil.getStartAndEndIP(subnetIP, subnetMask);
// 由起始+终止Ip 获取 在地址段内的 所有 IP
if(ipArr != null && ipArr.length == 2){
long startIpLong = IpUtil.ipToLong(ipArr[0]);
long endIpLong = IpUtil.ipToLong(ipArr[1]);
// ip 段跨度 较小时,按段 去 插入
if(endIpLong -startIpLong <= 5000){
List allRoomIp = IpUtil.getAllIpBySegment(ipArr[0], ipArr[1]);
if(allRoomIp != null && allRoomIp.size() > 0){
// 每次 读一段--存一段(先把验证数据去掉)
for (String ip : allRoomIp) {
OwnUsedAsset obj = new OwnUsedAsset();
obj.setArea(area);
obj.setIP(ip);
// obj.setIpLong(subnetMask);
obj.setRemark(remark);
list.add(obj);
obj = null;
}
}
}else{
// 当ip 跨度较大时,比如 mask = 8 时,此时一个 ip段就是 1670万+,解决策略是 每次 存1000 个
for(long i1=startIpLong; i1 <= endIpLong; i1++ ){
// 每次 读一段--存一段(先把验证数据去掉)
OwnUsedAsset obj = new OwnUsedAsset();
obj.setArea(area);
obj.setIP(IpUtil.longToIP(i1));
// obj.setIpLong(subnetMask);
obj.setRemark(remark);
list.add(obj);
obj = null;
if(list.size() >= 5000){
storeDB(list ,isInsert);
list = new ArrayList<>();//制空
}
}
// 最后的部分
storeDB(list ,isInsert);
list = new ArrayList<>();
}
}
// 每次循环检查一次,防止数据一次性过多
if(list.size() >= 5000){
storeDB(list ,isInsert);
list = new ArrayList<>();//制空
}
}
// 最后不满足 5000 行数据的也要插入
storeDB(list ,isInsert);
list = new ArrayList<>();//制空
}
} catch (Exception e) {
System.out.println("异常:"+e);
}
}
/**
*
* @param list
* @param isInsert
*/
public void storeDB(List list ,boolean isInsert){
if(isInsert){
OwnUsedAssetService.getInstance().insertAddressToDB(list);
}else{
OwnUsedAssetService.getInstance().insertAndUpdateInfo(list);
}
}
/**
* 导出 自用地址段表
* @param ids
* @return
*/
public Workbook writeExcel(long[] ids,Ioc ioc,HttpSession session, @Param("..") Pager pager,
@Param("search") String search,int state){
/*HSSFWorkbook wook = new HSSFWorkbook();
HSSFSheet sheet=wook.createSheet("自用地址段信息");
HSSFRow row=sheet.createRow(0); */
XSSFWorkbook work = new XSSFWorkbook();
XSSFSheet sheet = work.createSheet("自用地址段信息");
//在sheet中创建一行
XSSFRow row = sheet.createRow(0);
//在该行写入各种类型的数据
row.createCell(0).setCellValue("资产归属地");
row.createCell(1).setCellValue("资产IP");
if(state == 2){
row.createCell(2).setCellValue("最近一次发现时间");
row.createCell(3).setCellValue("备注");
sheet.setColumnWidth(2, (short)6500);
sheet.setColumnWidth(3, (short)15000);
}else{
row.createCell(2).setCellValue("备注");
sheet.setColumnWidth(2, (short)1500);
}
// 调整列宽
sheet.setColumnWidth(0, (short)2500);
sheet.setColumnWidth(1, (short)5000);
// sheet.setColumnWidth(2, (short)15000);
// 遍历数据库 取出数据,放入表格里
//在写入 日期格式的 数据需要进行特殊处理(这是一种 简单的处理方式)
int rowNum = 1;
List list = null;
BoolQueryBuilder qb = QueryBuilders.boolQuery();
pager.setPage(1);
pager.setPageSize(10000);
// 组合 模糊查询 should
if( state == 2){
QueryBuilder state2 = QueryBuilders.termQuery("state", 2);// 下线资产状态
qb.must(state2);
}
// 选择 条目的
if(search == null || search.equals("") ){
if(ids == null || ids.length == 0){
// 全查
list = OwnUsedAssetService.getInstance().getList(pager,qb).getData();
}else{
qb.must(QueryBuilders.inQuery("id", ids));
list = OwnUsedAssetService.getInstance().getList(pager,qb).getData();
}
}else{
// 说明 有 查询条件
// 1.已经选择 具体 信息,则继续 按照 ids 查询
if( ids != null && ids.length > 0){
qb.must(QueryBuilders.inQuery("id", ids));
list = OwnUsedAssetService.getInstance().getList(pager, qb).getData();
}else{
// 2.说明 有满足条件且 没有具体选择,则需要 模糊查询出符合条件的 所有信息
// 重新 设置 page
BoolQueryBuilder qbDFuzzy = QueryBuilders.boolQuery();
WildcardQueryBuilder ip = QueryBuilders.wildcardQuery("ip", "*"+search+"*");
WildcardQueryBuilder area = QueryBuilders.wildcardQuery("area", "*"+search+"*");
WildcardQueryBuilder remark = null;
if(state == 2){
remark = QueryBuilders.wildcardQuery("downLineRemark", "*"+search+"*");
WildcardQueryBuilder detectTime = QueryBuilders.wildcardQuery("detectTime", "*"+search+"*");
qbDFuzzy.should(ip).should(area).should(remark).should(detectTime);
}else{
remark = QueryBuilders.wildcardQuery("remark", "*"+search+"*");
qbDFuzzy.should(ip).should(area).should(remark);//
}
qb.must(qbDFuzzy);
list = OwnUsedAssetService.getInstance().getFuzzyList(pager, qb).getData();
}
}
if(state == 2){
for (OwnUsedAsset obj : list) {
XSSFRow rowInsert=sheet.createRow(rowNum);
rowInsert.createCell(0).setCellValue(obj.getArea());
rowInsert.createCell(1).setCellValue(obj.getIP());
rowInsert.createCell(2).setCellValue(obj.getDetectTime());
rowInsert.createCell(3).setCellValue(obj.getDownLineRemark());
rowNum ++;
}
}else{
for (OwnUsedAsset obj : list) {
XSSFRow rowInsert=sheet.createRow(rowNum);
rowInsert.createCell(0).setCellValue(obj.getArea());
rowInsert.createCell(1).setCellValue(obj.getIP());
//rowInsert.createCell(2).setCellValue(obj.getDetectTime());
rowInsert.createCell(2).setCellValue(obj.getRemark());
rowNum ++;
}
}
return work;
}
/**
* 与库中数据 进行对比,来判断是否 为 遗漏资产
* @param allIPLong
* @return
*//*
public Set getNoCountIPAddr(Set allIPLong ){
HashSet noCountIPSet = new HashSet();
if(allIPLong == null || allIPLong.size() ==0) return noCountIPSet;
for (Long ipLong : allIPLong) {
//true : 含有 false: 不含有
if(!OwnAssetCache.getInstance().checkDataInDB(ipLong)){
noCountIPSet.add(ipLong);
}
}
return noCountIPSet;
}*/
}
package com.test.ownasset.service.esService;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import org.apache.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolFilterBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.OrFilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.TermsQueryBuilder;
import org.elasticsearch.index.query.WildcardQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.nutz.ioc.loader.annotation.IocBean;
import com.test.commons.mvc.Pager;
import com.test.ownasset.cache.OwnUsedAssetCache;
import com.test.ownasset.data.AllAddressInProvice;
import com.test.ownasset.data.OwnUsedAsset;
import com.test.ownasset.util.DateUtil;
import com.test.ownasset.util.EsClientUtil;
import com.test.ownasset.util.IpUtil;
import com.test.ownasset.util.JsonUtil;
import com.test.util.es.ESClient;
/**
*
* @author Administrator
*
*/
@IocBean
public class OwnUsedAssetService {
private Logger logger = Logger.getLogger(OwnUsedAssetService.class);
private OwnUsedAssetService(){};
/**
* 单例模式
*/
private static OwnUsedAssetService service = null;
public static OwnUsedAssetService getInstance(){
if(service == null){
service = new OwnUsedAssetService();
}
return service;
}
private static String name ="ownused";
/**
* 添加单个数据 boolean exist = EsClientUtil.isExistIndex(name);
* @param obj
*/
public void insertAddressToDB(OwnUsedAsset obj){
ESClient client = EsClientUtil.getDefaultDelegateClient();
if(client != null){
//long id=new Date().getTime()*10000+new Random().nextInt(10000);
obj.setId(obj.getIpLong());
obj.setState(1);
obj.setCreateTime(System.currentTimeMillis());
obj.setUpdateTime(System.currentTimeMillis());
obj.setActionTime(DateUtil.getNowWeekNumber());
//obj.setDetectTime(DateUtil.getNowTimeStr());
String data = JsonUtil.toJson(obj);
client.prepareIndex(name,name)
.setId(String.valueOf(obj.getIpLong()))
.setSource(data)
.execute().actionGet();
// 添加到内存 OwnUsedAssetCache
OwnUsedAssetCache.ownUsedCache.put(obj.getIP(), DateUtil.getNowWeekNumber());
}
}
/**
* 批量添加全省地址段
* @param objList
*/
public void insertAddressToDB(List objList){
if(objList.size() == 0) return;
ESClient client = EsClientUtil.getDefaultDelegateClient();
if(client != null){
long actionTime = DateUtil.getNowWeekNumber();
// 开启批量
BulkRequestBuilder prepareBulk = client.prepareBulk();
for (OwnUsedAsset obj : objList) {
//long id=new Date().getTime()*10000+new Random().nextInt(10000);
obj.setId(obj.getIpLong());
obj.setState(1);
obj.setCreateTime(System.currentTimeMillis());
obj.setUpdateTime(System.currentTimeMillis());
obj.setActionTime(actionTime);
//obj.setDetectTime(DateUtil.getNowTimeStr());
String data = JsonUtil.toJson(obj);
prepareBulk.add(
client.prepareIndex(name,name)
.setId(String.valueOf(obj.getIpLong()))
.setSource(data)
);
// 添加到内存 OwnUsedAssetCache
OwnUsedAssetCache.ownUsedCache.put(obj.getIP(), actionTime);
}
prepareBulk.execute().actionGet();
logger.info("-->批量添加自用地址段:"+objList.size());
}
}
/**
* 批量添加 或更新数据
* 若是此IP,不存在则 添加,若是存在 则更新
* @param objList
*/
public void insertAndUpdateInfo(List objList){
if(objList == null || objList.size() == 0) return;
long start = System.currentTimeMillis();
ESClient client = EsClientUtil.getDefaultDelegateClient();
if(client != null){
// 先验证是否存在 getEntry(Map qbMap)
List insertList = new ArrayList();
HashMap dataMap = new HashMap();
for (OwnUsedAsset obj : objList) {
String docId = getDocIdUsedCache(obj.getIP());
if(docId.equals("")){
// 批量添加
insertList.add(obj);
}else{
dataMap.put(docId, obj.getRemark());
}
}
// 批量添加
if(insertList.size() > 0){
insertAddressToDB(insertList);
}
//批量更新
if(dataMap.size() > 0){
HashMap editMap = new HashMap<>();
editMap.put("updateTime",System.currentTimeMillis());
bulkUpdateState(dataMap,editMap);
}
}
long end = System.currentTimeMillis();
logger.info("批量添加 或更新数据:"+objList.size() +" 条-->"+"共耗时:"+(end - start)+" 毫秒!");
}
/**
* 更新 单条 -数据 通过id
* @param obj
*/
public void updateInfo(OwnUsedAsset obj){
if(obj == null) return;
boolean exist = EsClientUtil.isExistIndex(name);
if(exist){
ESClient client = EsClientUtil.getDefaultDelegateClient();
QueryBuilder qb = QueryBuilders.termQuery("id", obj.getId());
SearchResponse respose = client.prepareSearch(name)
.setQuery(qb)
.setSize(1)
.execute().actionGet();
SearchHit[] hits = respose.getHits().getHits();
if(hits.length > 0){
// 先获取文档ID
String docId = hits[0].getId();
// 对数据进行更新
if(obj.getState()!= 2){
obj.setState(1);
}
obj.setUpdateTime(System.currentTimeMillis());
String data = JsonUtil.toJson(obj);
// 进行操作
client.prepareIndex(name, name)
.setId(docId)
.setSource(data)
.execute()
.actionGet();
}
}
}
/**
* 删除信息
* @param ids
*/
public void deleteInfo(long[] ids){
if(ids == null || ids.length == 0) return;
boolean exist = EsClientUtil.isExistIndex(name);
if(exist){
ESClient client = EsClientUtil.getDefaultDelegateClient();
// 先查询 再 删除
QueryBuilder qb = QueryBuilders.inQuery("id", ids);
SearchResponse respose = client.prepareSearch(name)
.setTypes(name)
.setQuery(qb)
.setSize(Integer.MAX_VALUE)
.execute().actionGet();
SearchHit[] hits = respose.getHits().getHits();
if(hits.length > 0){
// 开启批量删除
BulkRequestBuilder bulkfresh = client.prepareBulk().setRefresh(true);
for (SearchHit searchHit : hits) {
DeleteRequest deleteRequest = new DeleteRequest(name, name, searchHit.getId());
bulkfresh.add(deleteRequest);
// 清楚缓存
OwnUsedAssetCache.ownUsedCache.remove(IpUtil.longToIP(Long.valueOf(searchHit.getId())));
}
// 执行
bulkfresh.execute().actionGet();
}
}
}
/**
* 当把下线资产删除时,应把 对应的资产库中的 state 由 2 -->1
* @param ids
*/
public void updateStateInfo(long[] ids){
if(ids == null || ids.length == 0) return;
boolean exist = EsClientUtil.isExistIndex(name);
if(exist){
ESClient client = EsClientUtil.getDefaultDelegateClient();
// 先查询 再 删除
QueryBuilder qb = QueryBuilders.inQuery("id", ids);
SearchResponse respose = client.prepareSearch(name)
.setTypes(name)
.setQuery(qb)
.setSize(Integer.MAX_VALUE)
.execute().actionGet();
SearchHit[] hits = respose.getHits().getHits();
if(hits.length > 0){
for (SearchHit searchHit : hits) {
OwnUsedAsset obj = new OwnUsedAsset();
Map source = searchHit.getSource();
String docId = searchHit.getId();
long id = Long.parseLong(source.get("id").toString());
long createTime = Long.parseLong(source.get("createTime").toString());
long actionTime = Long.parseLong(source.get("actionTime").toString());
long updateTime = Long.parseLong(source.get("updateTime").toString());
String ip = source.get("ip")== null ? "": source.get("ip").toString();
String area = source.get("area")== null ? "": source.get("area").toString();
String remark = source.get("remark")== null ? "": source.get("remark").toString();
// String downLineRemark = source.get("downLineRemark")== null ? "": source.get("downLineRemark").toString();
// String detectTime = source.get("detectTime")== null ? "": source.get("detectTime").toString();
obj.setId(id);
obj.setArea(area);
obj.setIP(ip);
obj.setRemark(remark);
obj.setActionTime(actionTime);
obj.setUpdateTime(updateTime);
obj.setCreateTime(createTime);
obj.setDetectTime("");
obj.setDownLineRemark("");
// 修改状态值
obj.setState(1);
String data = JsonUtil.toJson(obj);
//执行操作
client.prepareIndex(name,name)
.setId(docId)
.setSource(data)
.execute()
.actionGet();
}
}
}
}
/**
* 自用地址段 分页查询
* @param pager QueryBuilder qb = null 时 无条件查最新的
* @return
*/
public Pager getList( Pager pager,QueryBuilder qb){
// 初始化值
List list = new ArrayList();
long count = 0;
int pagerFrom = (pager.getPage()-1)*pager.getPageSize();
boolean exist = EsClientUtil.isExistIndex(name);
if(exist){
ESClient client = EsClientUtil.getDefaultDelegateClient();
SearchResponse respose = null;
if(qb != null){
respose = client.prepareSearch(name)
.addSort("createTime",SortOrder.DESC)
.setQuery(qb)
.setFrom(pagerFrom)
.setSize(pager.getPageSize())
.execute()
.actionGet();
}else{
respose = client.prepareSearch(name)
.addSort("createTime",SortOrder.DESC)
.setFrom(pagerFrom)
.setSize(pager.getPageSize())
.execute()
.actionGet();
}
SearchHits hits = respose.getHits();
count = hits.getTotalHits();
SearchHit[] dataArr = hits.getHits();
if(dataArr.length > 0){
for (SearchHit searchHit : dataArr) {
Map source = searchHit.getSource();
OwnUsedAsset obj = new OwnUsedAsset();
long id = Long.parseLong(source.get("id").toString());
String ip = source.get("ip")== null ? "": source.get("ip").toString();
String area = source.get("area")== null ? "": source.get("area").toString();
String remark = source.get("remark")== null ? "": source.get("remark").toString();
String downLineRemark = source.get("downLineRemark")== null ? "": source.get("downLineRemark").toString();
String detectTime = source.get("detectTime")== null ? "": source.get("detectTime").toString();
obj.setId(id);
obj.setArea(area);
obj.setIP(ip);
obj.setRemark(remark);
obj.setDetectTime(detectTime);
obj.setDownLineRemark(downLineRemark);
list.add(obj);
}
}
}
// 组装
pager.setRecords(count);
pager.setData(list);
return pager;
}
/**
* 全省地址段 模糊分页查询
* 只对三个字段进行 统计 分别是
* ip area remark
* @param pager
* @return
*/
public Pager getFuzzyList( Pager pager,QueryBuilder qb){
// 初始化值
List list = new ArrayList();
long count = 0;
int pagerFrom = (pager.getPage()-1)*pager.getPageSize();
boolean exist = EsClientUtil.isExistIndex(name);
if(exist){
ESClient client = EsClientUtil.getDefaultDelegateClient();
SearchResponse respose = client.prepareSearch(name)
.addSort("createTime",SortOrder.DESC)
.setFrom(pagerFrom)
.setQuery(qb)
.setSize(pager.getPageSize())
.execute()
.actionGet();
SearchHits hits = respose.getHits();
count = hits.getTotalHits();
SearchHit[] dataArr = hits.getHits();
if(dataArr.length > 0){
for (SearchHit searchHit : dataArr) {
Map source = searchHit.getSource();
OwnUsedAsset obj = new OwnUsedAsset();
long id = Long.parseLong(source.get("id").toString());
String ip = source.get("ip")== null ? "": source.get("ip").toString();
String area = source.get("area")== null ? "": source.get("area").toString();
String remark = source.get("remark")== null ? "": source.get("remark").toString();
String downLineRemark = source.get("downLineRemark")== null ? "": source.get("downLineRemark").toString();
String detectTime = source.get("detectTime")== null ? "": source.get("detectTime").toString();
obj.setId(id);
obj.setArea(area);
obj.setIP(ip);
obj.setRemark(remark);
obj.setDetectTime(detectTime);
obj.setDownLineRemark(downLineRemark);
list.add(obj);
}
}
}
// 组装
pager.setRecords(count);
pager.setData(list);
return pager;
}
/**
* 由qbMap 获取一条 满足条件的信息
* qbMap 不可为空,为空的话 return null;
* 获取 1条信息
* @param key
* @param value
* @return
*/
public OwnUsedAsset getEntry(Map qbMap){
OwnUsedAsset obj = null;
if(qbMap == null) return null;
boolean flag = EsClientUtil.isExistIndex(name);
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
BoolQueryBuilder qb = QueryBuilders.boolQuery();
Set keySet = qbMap.keySet();
for (Object key : keySet) {
QueryBuilders.termQuery(key.toString(), qbMap.get(key));
qb.must(QueryBuilders.termQuery(key.toString(), qbMap.get(key)));
}
SearchResponse respose = client.prepareSearch(name)
.setQuery(qb)
.setSize(1)
.execute().actionGet();
SearchHit[] hits = respose.getHits().getHits();
if(hits.length > 0){
Map source = hits[0].getSource();
obj= new OwnUsedAsset();
long id = Long.parseLong(source.get("id").toString());
long state = Long.parseLong(source.get("state").toString());
long createTime = Long.parseLong(source.get("createTime").toString());
long actionTime = Long.parseLong(source.get("actionTime").toString());
long updateTime = Long.parseLong(source.get("updateTime").toString());
String ip = source.get("ip")== null ? "": source.get("ip").toString();
String area = source.get("area")== null ? "": source.get("area").toString();
String remark = source.get("remark")== null ? "": source.get("remark").toString();
String downLineRemark = source.get("downLineRemark")== null ? "": source.get("downLineRemark").toString();
String detectTime = source.get("detectTime")== null ? "": source.get("detectTime").toString();
obj.setId(id);
obj.setArea(area);
obj.setIP(ip);
obj.setState(state);
obj.setDetectTime(detectTime);
obj.setDownLineRemark(downLineRemark);
obj.setActionTime(actionTime);
obj.setCreateTime(createTime);
obj.setUpdateTime(updateTime);
obj.setRemark(remark);
}
}
return obj;
}
/**
* 查询条件 由自己组装
* @param QueryBuilder qb
* @return
*/
public OwnUsedAsset getEntry(QueryBuilder qb){
OwnUsedAsset obj = null;
if(qb == null) return null;
boolean flag = EsClientUtil.isExistIndex(name);
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
SearchResponse respose = client.prepareSearch(name)
.setQuery(qb)
.setSize(1)
.execute().actionGet();
SearchHit[] hits = respose.getHits().getHits();
if(hits.length > 0){
Map source = hits[0].getSource();
obj= new OwnUsedAsset();
long id = Long.parseLong(source.get("id").toString());
String ip = source.get("ip")== null ? "": source.get("ip").toString();
String area = source.get("area")== null ? "": source.get("area").toString();
String remark = source.get("remark")== null ? "": source.get("remark").toString();
obj.setId(id);
obj.setArea(area);
obj.setIP(ip);
obj.setRemark(remark);
}
}
return obj;
}
/**
* 查询 ES
* 由ip 获取 自用资产里的 docId
* 若是 不存在: return "";
* @param ip
* @return
*/
public String getDocIdByIp(String ip){
String docId = "";
boolean flag = EsClientUtil.isExistIndex(name);
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
SearchResponse respose = client.prepareSearch(name)
.setQuery(QueryBuilders.termQuery("ip", ip))
.setSize(1)
.execute().actionGet();
SearchHit[] hits = respose.getHits().getHits();
if(hits.length > 0){
docId = hits[0].getId();
}
}
return docId;
}
/**
* 查询 cache
* 由ip 获取 自用资产里的 docId
* 若是 不存在: return "";
* @param ip
* @return
* {218.2.241.220=201712, 218.2.241.221=201712, 218.2.241.222=201712,
*/
public String getDocIdUsedCache(String ip){
String docId = "";
Long exist = OwnUsedAssetCache.ownUsedCache.get(ip);
if(exist != null){
docId = String.valueOf(IpUtil.ipToLong(ip));
}
return docId;
}
/**
* HashMap editMap = new HashMap<>();
editMap.put("state", 3);
editMap.put("actionTime", 666666);
* setDoc("actionTime", 10000)
* 每次只能改一个,后面的会覆盖前面的
* @param docId
*/
public void updateDataInField(String docId ,Map editMap){
boolean flag = EsClientUtil.isExistIndex(name);
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
client.prepareUpdate(name, name, docId)
.setDoc(editMap)
.execute().actionGet();
}
}
/**
* 批量更新 信息
* @param downLineID
*/
public void bulkUpdateState(Set downLineID ,Map editMap){
if(downLineID.size() == 0) return;
boolean flag = EsClientUtil.isExistIndex(name);
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
BulkRequestBuilder prepareBulk = client.prepareBulk();
for (String docId : downLineID) {
prepareBulk.add(client.prepareUpdate(name, name, docId).setDoc(editMap));
}
prepareBulk.execute().actionGet();
logger.info("更新自用资产:"+downLineID.size()+"条!");
}
}
/**
* 批量更新 信息
* @param downLineID
*/
public void bulkUpdateState(Map downLineID ,Map editMap){
if(downLineID.size() == 0) return;
boolean flag = EsClientUtil.isExistIndex(name);
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
BulkRequestBuilder prepareBulk = client.prepareBulk();
Set entrySet = downLineID.keySet();
for (String docId : entrySet) {
editMap.put("remark", downLineID.get(docId));
prepareBulk.add(client.prepareUpdate(name, name, docId).setDoc(editMap));
}
prepareBulk.execute().actionGet();
}
}
/**
* 获取超时 信息,为了防止 数据较大,采用 scroll 获取,避免 内存开销过大
* 1200 * size 每次取出的条数
* @param gaps
* @return
*/
public Set getOverTimeAssets(long gaps){
long start = System.currentTimeMillis();
HashSet downLineID = new HashSet();
boolean flag = EsClientUtil.isExistIndex(name);
long allSize = 0;
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
//long rangeTime = System.currentTimeMillis() -gaps;
// 久未活动 201652 201653 201701 201702 201703
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termQuery("state", 1));
QueryBuilder qb = QueryBuilders.rangeQuery("actionTime").lt(gaps);
SearchResponse respose = client.prepareSearch(name)
.setScroll(new TimeValue(60000))
.setQuery(boolQuery.must(qb))
.setSize(2000)
.setSearchType(SearchType.SCAN)
.execute().actionGet();
String scrollId = respose.getScrollId();
try {
while(true){
SearchResponse respose2 = client.prepareSearchScroll(scrollId)
.setScroll(new TimeValue(60000))
.execute().actionGet();
SearchHit[] hits = respose2.getHits().getHits();
if(hits.length == 0){
break;
}
//logger.info("每次获取超时自用资产:"+hits.length +"条!");
allSize+= hits.length;
for (SearchHit searchHit : hits) {
String docId = searchHit.getId();
downLineID.add(docId);
}
}
} catch (ElasticsearchException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
long end = System.currentTimeMillis();
logger.info("获取 超时未活动的资产耗时:"+(end - start)+"毫秒!"+downLineID.size() +"条!");
return downLineID;
}
/**
* 初始化 缓存 size * 分片数
* scroll 滑动游标
*/
public long initCache(int size){
logger.info("初始化自用资产,采用scroll游标滑动分段查询,每段获取:"+(12*size)+"条!");
boolean flag = EsClientUtil.isExistIndex(name);
long allSize = 0;
long start = System.currentTimeMillis();
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
SearchResponse respose = client.prepareSearch(name)
// 这个的意思是 加载的结果,在内存中所存放的时间,若 每次的 size 过大,而 TimeValue 又过小,这样会造成数据的丢失
.setScroll(new TimeValue(600000))// 保存一分钟
.setSize(size)// size*12分片
.setSearchType(SearchType.SCAN)// 不排序
.execute().actionGet();
String scrollId = respose.getScrollId();
try{
while(true){
SearchResponse resposeScroll = client.prepareSearchScroll(scrollId)
.setScroll(new TimeValue(600000))// 保存一分钟
.execute().actionGet();
SearchHit[] hits = resposeScroll.getHits().getHits();
if(hits == null || hits.length == 0){
break;// 跳出循环
}
allSize+= hits.length;
//System.out.println("allSize:"+allSize);
for (SearchHit searchHit : hits) {
Map source = searchHit.getSource();
long actionTime = Long.parseLong(source.get("actionTime").toString());
String ip = source.get("ip")== null ? "": source.get("ip").toString();
OwnUsedAssetCache.ownUsedCache.put(ip, actionTime);
}
}
long end = System.currentTimeMillis();
OwnUsedAssetCache.isOver = true;
logger.info("自用资产加载完毕,共加载:"+allSize +"共耗时:"+(end-start)+"毫秒!");
}catch(Exception e){
e.printStackTrace();
}
}else{
// 此索引暂不存在
logger.info("ES 中 indexName="+name+"的索引尚未存在，默认设置为 内存加载完成！");
OwnUsedAssetCache.isOver = true;
}
return allSize;
}
/**
* 批量更新 自用地址段-中遗漏-纳管ip 信息
* @param
* isHave = 2 // 表示 在 遗漏-纳管ip 中存在
*/
public void bulkUpdateOwnUsed(HashMap dataMap,int isHave){
if(dataMap.size() == 0) return;
boolean flag = EsClientUtil.isExistIndex("ownused");
if(flag){
ESClient client = EsClientUtil.getDefaultDelegateClient();
BulkRequestBuilder prepareBulk = client.prepareBulk();
// 然后进行批量更新
HashMap editMap = new HashMap();
editMap.put("state", isHave);// 表示 在 遗漏-纳管ip 中存在
Set> entrySet = dataMap.entrySet();
for (Entry entry : entrySet) {
String docId = entry.getKey();
String ipSegment = entry.getValue();
editMap.put("detectTime", ipSegment);// 存 归属 ip 段
prepareBulk.add(client.prepareUpdate("ownused", "ownused", docId).setDoc(editMap));
}
logger.info("更新自用资产段的遗漏-纳管ip");
prepareBulk.execute().actionGet();
}
}
public static void main(String[] args) {
OwnUsedAssetService.getInstance().initCache(2000);
//OwnUsedAssetService.getInstance().getOverTimeAssets(150*24*60*60*100L);
}
}
JsonUtil:
package com.test.ownasset.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JsonUtil {
/**
* 由对象转化成json 格式，添加到ES
*/
private static ObjectMapper objectMapper = new ObjectMapper();
public static String toJson(Object o){
try {
return objectMapper.writeValueAsString(o);
} catch (JsonProcessingException e) {
e.printStackTrace();
}
return "";
}
}
总结：里面包含 单条插入，批量插入，单条、批量更改，单、批量 删除，查询 与 模糊查询，滑动迭代取值等