当数据量达到千万条,mysql数据库不能查询和插入的情况下,使用es来替代mysql,对dao层代码进行改造。essql是需要付费的,本人写了2个通用的查询方法,一个针对聚合查询,一个针对非聚合查询。
针对聚合查询和非聚合查询,有如下几个实现类:
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.elasticsearch.search.sort.SortOrder;
public class ESCondition {
String index;
String type;
List rangeList = new ArrayList();//范围查询
String searchFields;//指定查询哪些字段(用户非聚合查询)
String countFields; //聚合统计
String sumFields;//聚合求和
String maxFields;
String minFields;
String avgFields;
String matchStr; //精确查询
String wildcardStr;//模糊查询
String nullStr;//查询条件里面某个字段为空并且不为空串筛选
String notNullStr;//查询条件里面某个字段不为空
Map> inField=new HashMap>();//in查询
String groupFields;
List sortList = new ArrayList();
Integer size=10000;//默认取出10000条
Integer from;
//需要进行正则匹配的字段
List regexList = new ArrayList();
//格式 去重字段1,去重字段2
String distinctFields;
public Integer getFrom() {
return from;
}
public void setFrom(Integer from) {
this.from = from;
}
public String getDistinctFields() {
return distinctFields;
}
public void setDistinctFields(String distinctFields) {
this.distinctFields = distinctFields;
}
public String getIndex() {
return index;
}
public void setIndex(String index) {
this.index = index;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public List getRegexList() {
return regexList;
}
public void addRegexList(String regexField,String regex) {
RegexParams rp = new RegexParams(regexField,regex);
regexList.add(rp);
}
public List getRangeList() {
return rangeList;
}
public void addRangeList(String rangeField,Object start,Object end) {
RangeParams rp = new RangeParams(rangeField,start,end);
rangeList.add(rp);
}
public void addRangeList(String rangeField,Object start,boolean includeLower,Object end,boolean includeUpper) {
RangeParams rp = new RangeParams(rangeField,start,includeLower,end,includeUpper);
rangeList.add(rp);
}
public String getSearchFields() {
return searchFields;
}
public void setSearchFields(String searchFields) {
this.searchFields = searchFields;
}
public String getCountFields() {
return countFields;
}
public void setCountFields(String countFields) {
this.countFields = countFields;
}
public String getSumFields() {
return sumFields;
}
public void setSumFields(String sumFields) {
this.sumFields = sumFields;
}
public String getMaxFields() {
return maxFields;
}
public void setMaxFields(String maxFields) {
this.maxFields = maxFields;
}
public String getMinFields() {
return minFields;
}
public void setMinFields(String minFields) {
this.minFields = minFields;
}
public String getAvgFields() {
return avgFields;
}
public void setAvgFields(String avgFields) {
this.avgFields = avgFields;
}
public String getMatchStr() {
return matchStr;
}
public void setMatchStr(String matchStr) {
this.matchStr = matchStr;
}
public String getGroupFields() {
return groupFields;
}
public void setGroupFields(String groupFields) {
this.groupFields = groupFields;
}
public List getSortList() {
return sortList;
}
public void addSortFields(String field, SortOrder so) {
SortParams sp = new SortParams(field,so);
sortList.add(sp);
}
public Integer getSize() {
return size;
}
public void setSize(Integer size) {
this.size = size;
}
public String getWildcardStr() {
return wildcardStr;
}
public void setWildcardStr(String wildcardStr) {
this.wildcardStr = wildcardStr;
}
public Map> getInField() {
return inField;
}
public void addInField(String field,String inValues) {
List inList = Arrays.asList(inValues.split(","));
inField.put(field, inList);
}
public void addInField(String field,List inList) {
inField.put(field, inList);
}
public String getNullStr() {
return nullStr;
}
public void setNullStr(String nullStr) {
this.nullStr = nullStr;
}
public String getNotNullStr() {
return notNullStr;
}
public void setNotNullStr(String notNullStr) {
this.notNullStr = notNullStr;
}
}
public class RangeParams {
String rangeField;
Object start;
boolean includeLower=false;
Object end;
boolean includeUpper=false;
String format;
public RangeParams(String rangeField, Object start, Object end) {
this.rangeField = rangeField;
this.start = start;
this.end = end;
}
public RangeParams(String rangeField, Object start, boolean includeLower, Object end, boolean includeUpper) {
this.rangeField = rangeField;
this.start = start;
this.includeLower = includeLower;
this.end = end;
this.includeUpper = includeUpper;
}
public String getRangeField() {
return rangeField;
}
public void setRangeField(String rangeField) {
this.rangeField = rangeField;
}
public Object getStart() {
return start;
}
public void setStart(Object start) {
this.start = start;
}
public Object getEnd() {
return end;
}
public void setEnd(Object end) {
this.end = end;
}
public String getFormat() {
return format;
}
public void setFormat(String format) {
this.format = format;
}
public boolean isIncludeLower() {
return includeLower;
}
public void setIncludeLower(boolean includeLower) {
this.includeLower = includeLower;
}
public boolean isIncludeUpper() {
return includeUpper;
}
public void setIncludeUpper(boolean includeUpper) {
this.includeUpper = includeUpper;
}
}
public class RegexParams {
String regexfiled;
String regex;
public RegexParams(String regexfiled, String regex) {
this.regexfiled = regexfiled;
this.regex = regex;
}
public String getRegexfiled() {
return regexfiled;
}
public void setRegexfiled(String regexfiled) {
this.regexfiled = regexfiled;
}
public String getRegex() {
return regex;
}
public void setRegex(String regex) {
this.regex = regex;
}
}
import org.elasticsearch.search.sort.SortOrder;
public class SortParams {
String sortField;
SortOrder sortOrder;
public SortParams(String field, SortOrder so) {
this.sortField = field;
this.sortOrder = so;
}
public String getSortField() {
return sortField;
}
public void setSortField(String sortField) {
this.sortField = sortField;
}
public SortOrder getSortOrder() {
return sortOrder;
}
public void setSortOrder(SortOrder sortOrder) {
this.sortOrder = sortOrder;
}
}
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
public class GroupBy {
private AggregationBuilder termsBuilder;
//记录上次的子builder
private AggregationBuilder lastGroupBuilder;
public void addSubAgg(String aggName, String fieldName) {
if(termsBuilder == null) {
termsBuilder = AggregationBuilders.terms(aggName).field(fieldName).size(10000);
lastGroupBuilder = termsBuilder;
} else {
AggregationBuilder n= AggregationBuilders.terms(aggName).field(fieldName).size(10000);
lastGroupBuilder.subAggregation(n);
lastGroupBuilder=n;
}
}
// public void add
public void addSumAgg(String aggName, String fieldName) {
SumAggregationBuilder builder = AggregationBuilders.sum(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
public void addCountAgg(String aggName, String fieldName) {
ValueCountAggregationBuilder builder = AggregationBuilders.count(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
public void addAvgAgg(String aggName, String fieldName) {
AvgAggregationBuilder builder = AggregationBuilders.avg(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
public void addMinAgg(String aggName, String fieldName) {
MinAggregationBuilder builder = AggregationBuilders.min(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
public void addMaxAgg(String aggName, String fieldName) {
MaxAggregationBuilder builder = AggregationBuilders.max(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
public void addStatsAgg(String aggName, String fieldName) {
StatsAggregationBuilder builder = AggregationBuilders.stats(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
public void addCardinalityAgg(String aggName, String fieldName) {
CardinalityAggregationBuilder builder = AggregationBuilders.cardinality(aggName).field(fieldName);
lastGroupBuilder = lastGroupBuilder.subAggregation(builder);
}
//取出非聚合查询的字段,与聚合查询的字段合并
public AggregationBuilder getTermsBuilder() {
AggregationBuilder top = AggregationBuilders.topHits("top").explain(true);
// .size(3).from(0);
lastGroupBuilder = lastGroupBuilder.subAggregation(top);
return termsBuilder;
}
public void setTermsBuilder(AggregationBuilder termsBuilder) {
this.termsBuilder = termsBuilder;
}
}
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
import org.elasticsearch.search.aggregations.metrics.max.Max;
import org.elasticsearch.search.aggregations.metrics.min.Min;
import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import com.alibaba.fastjson.JSONObject;
public class AggUtil {
private static Client client = null;
/**
*需要拿到一个客户端,在这里我们用的自己公司开发的客户端连接池。
*
*
*/
public static List getAggDocs(ESCondition c) {
ElasticHelper helper = ElasticClientPool.getInstance();
List results=null;
GroupBy gb = new GroupBy();
List rangeList = c.getRangeList();
List sortList = c.getSortList();
List regexList = c.getRegexList();
Map> inMap = c.getInField();
String groupArr[] = null,countArr[] = null,maxArr[] = null ,minArr[] = null ,sumArr[] = null ,avgArr[] = null ;
try {
TransportClient client = helper.getClient();
SearchRequestBuilder searchRequest = client.prepareSearch(c.getIndex());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if(rangeList != null && rangeList.size() != 0) {
for(int i=0 ; i < rangeList.size() ; i++) {
RangeParams rp = rangeList.get(i);
boolQuery.must(
QueryBuilders.rangeQuery(rp.getRangeField())
.from(rp.getStart())
.to(rp.getEnd())
.includeLower(true)
.includeUpper(true));
}
}
if(regexList != null && regexList.size() != 0) {
for(int i=0 ; i < regexList.size() ; i++) {
RegexParams rp = regexList.get(i);
boolQuery.must(
QueryBuilders.regexpQuery(rp.getRegexfiled(),rp.getRegex()));
}
}
if (StringUtils.isNotEmpty(c.getMatchStr())) {
for (String s : c.getMatchStr().split(",")) {
String[] ss = s.split("=");
if(ss.length > 1){
boolQuery.must(QueryBuilders.matchQuery(s.split("=")[0], s.split("=")[1]));
}
}
}
//in查询
if (inMap.size() != 0) {
for(Map.Entry> entry : inMap.entrySet() ) {
boolQuery.must(QueryBuilders.termsQuery(entry.getKey(), entry.getValue()));
}
}
searchRequest.setQuery(boolQuery);
//查询哪些字段
if (StringUtils.isNotEmpty(c.getSearchFields())) {
searchRequest.setFetchSource(c.getSearchFields().split(","), null);
searchRequest.setFetchSource(true);
}
//查询分组
if (StringUtils.isNotEmpty(c.getGroupFields())) {
groupArr = c.getGroupFields().split(",");
for(String kvField : groupArr) {
String kv[] = kvField.split(":");
gb.addSubAgg(kv[0], kv[1]);
}
}
//count统计
if (StringUtils.isNotEmpty(c.getCountFields())) {
countArr = c.getCountFields().split(",");
for(String kvField : countArr) {
String kv[] = kvField.split(":");
gb.addCountAgg(kv[0], kv[1]);
}
}
//sum统计
if (StringUtils.isNotEmpty(c.getSumFields())) {
sumArr = c.getSumFields().split(",");
for(String kvField : sumArr) {
String kv[] = kvField.split(":");
gb.addSumAgg(kv[0], kv[1]);
}
}
//max统计
if (StringUtils.isNotEmpty(c.getMaxFields())) {
maxArr = c.getMaxFields().split(",");
for(String kvField : maxArr) {
String kv[] = kvField.split(":");
gb.addMaxAgg(kv[0], kv[1]);
}
}
//min统计
if (StringUtils.isNotEmpty(c.getMinFields())) {
minArr = c.getMinFields().split(",");
for(String kvField : minArr) {
String kv[] = kvField.split(":");
gb.addMinAgg(kv[0], kv[1]);
}
}
//avg统计
if (StringUtils.isNotEmpty(c.getAvgFields())) {
avgArr = c.getAvgFields().split(",");
for(String kvField : avgArr) {
String kv[] = kvField.split(":");
gb.addAvgAgg(kv[0], kv[1]);
}
}
if (sortList != null && sortList.size()!=0) {
for(int i=0 ; i < sortList.size() ; i++) {
SortParams sp = sortList.get(i);
searchRequest.addSort(sp.getSortField(), sp.getSortOrder());
}
}
if(c.getSize() != null) {
searchRequest.setSize(c.getSize());
}
SearchResponse response = null;
searchRequest.addAggregation(gb.getTermsBuilder());
response= searchRequest.setSearchType(SearchType.QUERY_THEN_FETCH).execute().actionGet();
Map result = response.getAggregations().asMap();
results = groupTree(groupArr,0,result,countArr,maxArr,minArr,sumArr,avgArr);
} catch (Exception e) {
e.printStackTrace();
logger.error(e.getMessage(),e);
}finally {
ElasticClientPool.freeElasticClient(helper);
}
return results;
}
private static List groupTree(String[] groupArr,int index,Map result, String[] countArr, String[] maxArr, String[] minArr, String[] sumArr, String[] avgArr) {
List jsonList = new ArrayList();
if(index==groupArr.length-1) {
Terms terms = (Terms) /*(StringTerms)*/ result.get(groupArr[index].split(":")[0]);
for(Terms.Bucket bt : terms.getBuckets()) {
//先取出基础字段
TopHits topHits = bt.getAggregations().get("top");
//聚合只需取每组第一条封装
SearchHit first = topHits.getHits().getHits()[0];
JSONObject jsonObject = JSONObject.parseObject(first.getSourceAsString());
jsonObject.put("docCount", bt.getDocCount());
if(countArr != null){
for(String countField : countArr) {
String kv[]=countField.split(":");
ValueCount count = bt.getAggregations().get(kv[0]);
jsonObject.put(kv[0], count.getValue());
}
}
if(maxArr != null) {
for(String maxField : maxArr) {
String kv[]=maxField.split(":");
Max max = bt.getAggregations().get(kv[0]);
jsonObject.put(kv[0], max.getValue());
}
}
if(minArr != null) {
for(String minField : minArr) {
String kv[]=minField.split(":");
Min min = bt.getAggregations().get(kv[0]);
jsonObject.put(kv[0], min.getValue());
}
}
if(sumArr != null) {
for(String sumField : sumArr) {
String kv[]=sumField.split(":");
Sum sum = bt.getAggregations().get(kv[0]);
jsonObject.put(kv[0], sum.getValue());
}
}
if(avgArr != null) {
for(String avgField : avgArr) {
String kv[]=avgField.split(":");
Avg avg = bt.getAggregations().get(kv[0]);
jsonObject.put(kv[0], avg.getValue());
}
}
jsonList.add(jsonObject);
}
} else {
Terms terms = (Terms) result.get(groupArr[index].split(":")[0]);
if(terms != null) {
for(Terms.Bucket bt : terms.getBuckets()) {
Map result2 = bt.getAggregations().asMap();
List jsonList2 = groupTree(groupArr, index+1, result2, countArr, maxArr, minArr, sumArr, avgArr);
jsonList.addAll(jsonList2);
}
} else {
return new ArrayList();
}
}
return jsonList;
}
}
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.search.MultiMatchQuery.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms.Bucket;
import org.elasticsearch.search.sort.SortOrder;
import com.alibaba.fastjson.JSONObject;
public class ListUtil {
public static List getListDocs(ESCondition c) {
ElasticHelper helper = ElasticClientPool.getInstance();
TransportClient client = helper.getClient();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
SearchRequestBuilder searchRequest = client.prepareSearch(c.getIndex());
List rangeList = c.getRangeList();
List sortList = c.getSortList();
List regexList = c.getRegexList();
Map> inMap = c.getInField();
if(rangeList != null && rangeList.size() != 0) {
for(int i=0 ; i < rangeList.size() ; i++) {
RangeParams rp = rangeList.get(i);
boolQuery.must(
QueryBuilders.rangeQuery(rp.getRangeField())
.from(rp.getStart())
.to(rp.getEnd())
.includeLower(rp.isIncludeLower())
.includeUpper(rp.isIncludeUpper()));
}
}
//field is null
if (StringUtils.isNotEmpty(c.getNullStr())) {
for (String field : c.getNullStr().split(",")) {
boolQuery.must(QueryBuilders.existsQuery(field)).mustNot(QueryBuilders.regexpQuery(field, ".+"));
}
}
if (StringUtils.isNotEmpty(c.getNotNullStr())) {
for (String field : c.getNotNullStr().split(",")) {
boolQuery.must(QueryBuilders.existsQuery(field)).must(QueryBuilders.regexpQuery(field, ".+"));
}
}
if(regexList != null && regexList.size() != 0) {
for(int i=0 ; i < regexList.size() ; i++) {
RegexParams rp = regexList.get(i);
boolQuery.must(
QueryBuilders.regexpQuery(rp.getRegexfiled(),rp.getRegex()));
}
}
if (StringUtils.isNotEmpty(c.getMatchStr())) {
for (String s : c.getMatchStr().split(",")) {
String[] ss = s.split(":");
if(ss.length > 1){
boolQuery.must(QueryBuilders.matchQuery(ss[0], ss[1]));
}
}
}
if (inMap.size() != 0) {
for(Map.Entry> entry : inMap.entrySet() ) {
boolQuery.must(QueryBuilders.termsQuery(entry.getKey(), entry.getValue()));
}
}
//模糊查询
if (StringUtils.isNotEmpty(c.getWildcardStr())) {
BoolQueryBuilder or = QueryBuilders.boolQuery();
for (String s : c.getWildcardStr().split(",")) {
String[] ss = s.split(":");
if(ss.length > 1){
or = or.should(QueryBuilders.wildcardQuery(ss[0], "*"+ss[1]+"*"));
}
}
boolQuery.must(or);
}
searchRequest.setQuery(boolQuery);
if (StringUtils.isNotEmpty(c.getSearchFields())) {
searchRequest.setFetchSource(c.getSearchFields().split(","), null);
}
if (sortList != null && sortList.size()!=0) {
for(int i=0 ; i < sortList.size() ; i++) {
SortParams sp = sortList.get(i);
searchRequest.addSort(sp.getSortField(), sp.getSortOrder());
}
}
searchRequest.setFetchSource(true);
if(c.getFrom()==null &&c.getSize()!=null){
searchRequest.setSize(c.getSize());
}else if (c.getFrom()!=null &&c.getSize()!=null){
searchRequest.setFrom(c.getFrom()).setSize(c.getSize());
}
List sourceList = null;
try {
SearchResponse response = searchRequest
.execute()
.actionGet();
SearchHits shs = response.getHits();
sourceList = new ArrayList();
for (SearchHit sh : shs) {
JSONObject jsonObject = JSONObject.parseObject(sh.getSourceAsString());
jsonObject.put("id", sh.getId());
sourceList.add(jsonObject);
}
} catch (Exception e) {
e.printStackTrace();
}
return sourceList;
}
}
ES数据源改造就这些,能对付大部分sql查询,除了一些三元表达式,特殊函数。唯一的不足就是一次只能查最大10000条,这个要根据自己的数据量,在建表的时候设置上限。查询出来的结果返回的是一个json数据格式的字符串列表,自己再逐个取出需要的字段回填到对象就OK了