ES 实战
ES 实战
更多干货
分布式实战（干货）
spring cloud 实战（干货）
mybatis 实战（干货）
spring boot 实战（干货）
React 入门实战（干货）
构建中小型互联网企业架构（干货）
python 学习持续更新
ElasticSearch 笔记
kafka storm 实战 (干货)
scala 学习持续更新
RPC
深度学习
一、概述
项目是基于搜房子为背景的一个小项目，综合对ES 等技术进行综合的应用。
由于相关的技术点比较多具体不一一列举，以下将设计到es部分的代码进行展示，有兴趣的可以查看代码
项目地址: https://github.com/csy512889371/learndemo/tree/master/soufang
项目设计到的技术：
spring boot
spring security 用户认证
jpa
junit test h2
文件上传、上传到七牛云存储
阿里云短信配置
elastic search ** 异步创建索引 ** 搜索提示 ** 聚合统计 ** 结合百度地图点聚合 ** 百度LBS
ELK logstash
kafka
Thymeleaf
email邮件的报警
spring schedule 监控任务
二、ES相关代码
1、异步构建索引及其操作
使用kafka 实现消息中间件
# kafka
spring.kafka.bootstrap-servers=127.0.0.1:9092
spring.kafka.consumer.group-id=xunwu2、ElasticSearchConfig 配置类
@Configuration
public class ElasticSearchConfig {
@Value("${elasticsearch.host}")
private String esHost;
@Value("${elasticsearch.port}")
private int esPort;
@Value("${elasticsearch.cluster.name}")
private String esName;
@Bean
public TransportClient esClient() throws UnknownHostException {
Settings settings = Settings.builder()
.put("cluster.name", this.esName)
// .put("cluster.name", "elasticsearch")
.put("client.transport.sniff", true)
.build();
InetSocketTransportAddress master = new InetSocketTransportAddress(
InetAddress.getByName(esHost), esPort
// InetAddress.getByName("10.99.207.76"), 8999
);
TransportClient client = new PreBuiltTransportClient(settings)
.addTransportAddress(master);
return client;
}
}elasticsearch.cluster.name=xunwu
elasticsearch.host=127.0.0.1
elasticsearch.port=93003、house_index_mapping.json 定义es中的数据模型
{
"settings": {
"number_of_replicas": 0,
"number_of_shards": 5,
"index.store.type": "niofs",
"index.query.default_field": "title",
"index.unassigned.node_left.delayed_timeout": "5m"
},
"mappings": {
"house": {
"dynamic": "strict",
"_all": {
"enabled": false
},
"properties": {
"houseId": {
"type": "long"
},
"title": {
"type": "text",
"index": "analyzed",
"analyzer": "ik_smart",
"search_analyzer": "ik_smart"
},
"price": {
"type": "integer"
},
"area": {
"type": "integer"
},
"createTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"lastUpdateTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"cityEnName": {
"type": "keyword"
},
"regionEnName": {
"type": "keyword"
},
"direction": {
"type": "integer"
},
"distanceToSubway": {
"type": "integer"
},
"subwayLineName": {
"type": "keyword"
},
"subwayStationName": {
"type": "keyword"
},
"tags": {
"type": "text"
},
"street": {
"type": "keyword"
},
"district": {
"type": "keyword"
},
"description": {
"type": "text",
"index": "analyzed",
"analyzer": "ik_smart",
"search_analyzer": "ik_smart"
},
"layoutDesc" : {
"type": "text",
"index": "analyzed",
"analyzer": "ik_smart",
"search_analyzer": "ik_smart"
},
"traffic": {
"type": "text",
"index": "analyzed",
"analyzer": "ik_smart",
"search_analyzer": "ik_smart"
},
"roundService": {
"type": "text",
"index": "analyzed",
"analyzer": "ik_smart",
"search_analyzer": "ik_smart"
},
"rentWay": {
"type": "integer"
},
"suggest": {
"type": "completion"
},
"location": {
"type": "geo_point"
}
}
}
}
}4、HouseIndexTemplate 与es中定义的type 中的字段一一对应
public class HouseIndexTemplate {
private Long houseId;
private String title;
private int price;
private int area;
private Date createTime;
private Date lastUpdateTime;
private String cityEnName;
private String regionEnName;
private int direction;
private int distanceToSubway;
private String subwayLineName;
private String subwayStationName;
private String street;
private String district;
private String description;
private String layoutDesc;
private String traffic;
private String roundService;
private int rentWay;
private List tags;
private List suggest;
private BaiduMapLocation location;
... get set 方法
} 5、HouseIndexKey 常量类
/**
* 索引关键词统一定义
*
*/
public class HouseIndexKey {
public static final String HOUSE_ID = "houseId";
public static final String TITLE = "title";
public static final String PRICE = "price";
public static final String AREA = "area";
public static final String CREATE_TIME = "createTime";
public static final String LAST_UPDATE_TIME = "lastUpdateTime";
public static final String CITY_EN_NAME = "cityEnName";
public static final String REGION_EN_NAME = "regionEnName";
public static final String DIRECTION = "direction";
public static final String DISTANCE_TO_SUBWAY = "distanceToSubway";
public static final String STREET = "street";
public static final String DISTRICT = "district";
public static final String DESCRIPTION = "description";
public static final String LAYOUT_DESC = "layoutDesc";
public static final String TRAFFIC = "traffic";
public static final String ROUND_SERVICE = "roundService";
public static final String RENT_WAY = "rentWay";
public static final String SUBWAY_LINE_NAME = "subwayLineName";
public static final String SUBWAY_STATION_NAME = "subwayStationName";
public static final String TAGS = "tags";
public static final String AGG_DISTRICT = "agg_district";
public static final String AGG_REGION = "agg_region";
}6、搜索补全
HouseSuggest
public class HouseSuggest {
private String input;
private int weight = 10; // 默认权重
}
"suggest": {
"type": "completion"
}
@Override
public ServiceResult> suggest(String prefix) {
CompletionSuggestionBuilder suggestion = SuggestBuilders.completionSuggestion("suggest").prefix(prefix).size(5);
SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.addSuggestion("autocomplete", suggestion);
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.suggest(suggestBuilder);
logger.debug(requestBuilder.toString());
SearchResponse response = requestBuilder.get();
Suggest suggest = response.getSuggest();
if (suggest == null) {
return ServiceResult.of(new ArrayList<>());
}
Suggest.Suggestion result = suggest.getSuggestion("autocomplete");
int maxSuggest = 0;
Set suggestSet = new HashSet<>();
for (Object term : result.getEntries()) {
if (term instanceof CompletionSuggestion.Entry) {
CompletionSuggestion.Entry item = (CompletionSuggestion.Entry) term;
if (item.getOptions().isEmpty()) {
continue;
}
for (CompletionSuggestion.Entry.Option option : item.getOptions()) {
String tip = option.getText().string();
if (suggestSet.contains(tip)) {
continue;
}
suggestSet.add(tip);
maxSuggest++;
}
}
if (maxSuggest > 5) {
break;
}
}
List suggests = Lists.newArrayList(suggestSet.toArray(new String[]{}));
return ServiceResult.of(suggests);
} 7、聚合分析
@Override
public ServiceResult aggregateDistrictHouse(String cityEnName, String regionEnName, String district) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
.filter(QueryBuilders.termQuery(HouseIndexKey.CITY_EN_NAME, cityEnName))
.filter(QueryBuilders.termQuery(HouseIndexKey.REGION_EN_NAME, regionEnName))
.filter(QueryBuilders.termQuery(HouseIndexKey.DISTRICT, district));
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addAggregation(
AggregationBuilders.terms(HouseIndexKey.AGG_DISTRICT)
.field(HouseIndexKey.DISTRICT)
).setSize(0);
logger.debug(requestBuilder.toString());
SearchResponse response = requestBuilder.get();
if (response.status() == RestStatus.OK) {
Terms terms = response.getAggregations().get(HouseIndexKey.AGG_DISTRICT);
if (terms.getBuckets() != null && !terms.getBuckets().isEmpty()) {
return ServiceResult.of(terms.getBucketByKey(district).getDocCount());
}
} else {
logger.warn("Failed to Aggregate for " + HouseIndexKey.AGG_DISTRICT);
}
return ServiceResult.of(0L);
} 8、优化
打印查询语句
logger.debug(requestBuilder.toString());查询的时候 加 explain=true 来查看搜索排序规则
/index/type/_search?explain=true对关键字段 手动增加权重
// boolQuery.must(
// QueryBuilders.matchQuery(HouseIndexKey.TITLE, rentSearch.getKeywords())
// .boost(2.0f)
// );限制返回的字段 setFetchSource(HouseIndexKey.HOUSE_ID, null)
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addSort(
HouseSort.getSortKey(rentSearch.getOrderBy()),
SortOrder.fromString(rentSearch.getOrderDirection())
)
.setFrom(rentSearch.getStart())
.setSize(rentSearch.getSize())
.setFetchSource(HouseIndexKey.HOUSE_ID, null);设置索引刷新时间
{
"transient": {
"index.refresh_interval": "30s"
}
}
配置文件中 加入
index.refresh_interval: 30s
现在删除时候指定索引名称 防止误删除
{
"transient": {
"action.destructive_requires_name": true
}
}集群恢复设置
discovery.zen.fd.ping_interval: 10s
discovery.zen.fd.ping_timeout: 120s
discovery.zen.fd.ping_retries: 10不同的节点 做各自的事情
指挥节点
node.master: true
node.data: false
数据节点
node.master:false
node.data: true
http.enable:false
负载均衡节点
node.master: false
node.data: false
内存设置 32G 以内 且不能操作50%
硬盘使用 ssd
批量操作使用bulk 提升qps
9、server 方法全
@Service
public class SearchServiceImpl implements ISearchService {
private static final Logger logger = LoggerFactory.getLogger(ISearchService.class);
private static final String INDEX_NAME = "xunwu";
private static final String INDEX_TYPE = "house";
private static final String INDEX_TOPIC = "house_build";
@Autowired
private HouseRepository houseRepository;
@Autowired
private HouseDetailRepository houseDetailRepository;
@Autowired
private HouseTagRepository tagRepository;
@Autowired
private SupportAddressRepository supportAddressRepository;
@Autowired
private IAddressService addressService;
@Autowired
private ModelMapper modelMapper;
@Autowired
private TransportClient esClient;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private KafkaTemplate kafkaTemplate;
@KafkaListener(topics = INDEX_TOPIC)
private void handleMessage(String content) {
try {
HouseIndexMessage message = objectMapper.readValue(content, HouseIndexMessage.class);
switch (message.getOperation()) {
case HouseIndexMessage.INDEX:
this.createOrUpdateIndex(message);
break;
case HouseIndexMessage.REMOVE:
this.removeIndex(message);
break;
default:
logger.warn("Not support message content " + content);
break;
}
} catch (IOException e) {
logger.error("Cannot parse json for " + content, e);
}
}
private void createOrUpdateIndex(HouseIndexMessage message) {
Long houseId = message.getHouseId();
House house = houseRepository.findOne(houseId);
if (house == null) {
logger.error("Index house {} dose not exist!", houseId);
this.index(houseId, message.getRetry() + 1);
return;
}
HouseIndexTemplate indexTemplate = new HouseIndexTemplate();
modelMapper.map(house, indexTemplate);
HouseDetail detail = houseDetailRepository.findByHouseId(houseId);
if (detail == null) {
// TODO 异常情况
}
modelMapper.map(detail, indexTemplate);
SupportAddress city = supportAddressRepository.findByEnNameAndLevel(house.getCityEnName(), SupportAddress.Level.CITY.getValue());
SupportAddress region = supportAddressRepository.findByEnNameAndLevel(house.getRegionEnName(), SupportAddress.Level.REGION.getValue());
String address = city.getCnName() + region.getCnName() + house.getStreet() + house.getDistrict() + detail.getDetailAddress();
ServiceResult location = addressService.getBaiduMapLocation(city.getCnName(), address);
if (!location.isSuccess()) {
this.index(message.getHouseId(), message.getRetry() + 1);
return;
}
indexTemplate.setLocation(location.getResult());
List tags = tagRepository.findAllByHouseId(houseId);
if (tags != null && !tags.isEmpty()) {
List tagStrings = new ArrayList<>();
tags.forEach(houseTag -> tagStrings.add(houseTag.getName()));
indexTemplate.setTags(tagStrings);
}
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME).setTypes(INDEX_TYPE)
.setQuery(QueryBuilders.termQuery(HouseIndexKey.HOUSE_ID, houseId));
logger.debug(requestBuilder.toString());
SearchResponse searchResponse = requestBuilder.get();
boolean success;
long totalHit = searchResponse.getHits().getTotalHits();
if (totalHit == 0) {
success = create(indexTemplate);
} else if (totalHit == 1) {
String esId = searchResponse.getHits().getAt(0).getId();
success = update(esId, indexTemplate);
} else {
success = deleteAndCreate(totalHit, indexTemplate);
}
ServiceResult serviceResult = addressService.lbsUpload(location.getResult(), house.getStreet() + house.getDistrict(),
city.getCnName() + region.getCnName() + house.getStreet() + house.getDistrict(),
message.getHouseId(), house.getPrice(), house.getArea());
if (!success || !serviceResult.isSuccess()) {
this.index(message.getHouseId(), message.getRetry() + 1);
} else {
logger.debug("Index success with house " + houseId);
}
}
private void removeIndex(HouseIndexMessage message) {
Long houseId = message.getHouseId();
DeleteByQueryRequestBuilder builder = DeleteByQueryAction.INSTANCE
.newRequestBuilder(esClient)
.filter(QueryBuilders.termQuery(HouseIndexKey.HOUSE_ID, houseId))
.source(INDEX_NAME);
logger.debug("Delete by query for house: " + builder);
BulkByScrollResponse response = builder.get();
long deleted = response.getDeleted();
logger.debug("Delete total " + deleted);
ServiceResult serviceResult = addressService.removeLbs(houseId);
if (!serviceResult.isSuccess() || deleted <= 0) {
logger.warn("Did not remove data from es for response: " + response);
// 重新加入消息队列
this.remove(houseId, message.getRetry() + 1);
}
}
@Override
public void index(Long houseId) {
this.index(houseId, 0);
}
private void index(Long houseId, int retry) {
if (retry > HouseIndexMessage.MAX_RETRY) {
logger.error("Retry index times over 3 for house: " + houseId + " Please check it!");
return;
}
HouseIndexMessage message = new HouseIndexMessage(houseId, HouseIndexMessage.INDEX, retry);
try {
kafkaTemplate.send(INDEX_TOPIC, objectMapper.writeValueAsString(message));
} catch (JsonProcessingException e) {
logger.error("Json encode error for " + message);
}
}
private boolean create(HouseIndexTemplate indexTemplate) {
if (!updateSuggest(indexTemplate)) {
return false;
}
try {
IndexResponse response = this.esClient.prepareIndex(INDEX_NAME, INDEX_TYPE)
.setSource(objectMapper.writeValueAsBytes(indexTemplate), XContentType.JSON).get();
logger.debug("Create index with house: " + indexTemplate.getHouseId());
if (response.status() == RestStatus.CREATED) {
return true;
} else {
return false;
}
} catch (JsonProcessingException e) {
logger.error("Error to index house " + indexTemplate.getHouseId(), e);
return false;
}
}
private boolean update(String esId, HouseIndexTemplate indexTemplate) {
if (!updateSuggest(indexTemplate)) {
return false;
}
try {
UpdateResponse response = this.esClient.prepareUpdate(INDEX_NAME, INDEX_TYPE, esId).setDoc(objectMapper.writeValueAsBytes(indexTemplate), XContentType.JSON).get();
logger.debug("Update index with house: " + indexTemplate.getHouseId());
if (response.status() == RestStatus.OK) {
return true;
} else {
return false;
}
} catch (JsonProcessingException e) {
logger.error("Error to index house " + indexTemplate.getHouseId(), e);
return false;
}
}
private boolean deleteAndCreate(long totalHit, HouseIndexTemplate indexTemplate) {
DeleteByQueryRequestBuilder builder = DeleteByQueryAction.INSTANCE
.newRequestBuilder(esClient)
.filter(QueryBuilders.termQuery(HouseIndexKey.HOUSE_ID, indexTemplate.getHouseId()))
.source(INDEX_NAME);
logger.debug("Delete by query for house: " + builder);
BulkByScrollResponse response = builder.get();
long deleted = response.getDeleted();
if (deleted != totalHit) {
logger.warn("Need delete {}, but {} was deleted!", totalHit, deleted);
return false;
} else {
return create(indexTemplate);
}
}
@Override
public void remove(Long houseId) {
this.remove(houseId, 0);
}
@Override
public ServiceMultiResult query(RentSearch rentSearch) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.filter(
QueryBuilders.termQuery(HouseIndexKey.CITY_EN_NAME, rentSearch.getCityEnName())
);
if (rentSearch.getRegionEnName() != null && !"*".equals(rentSearch.getRegionEnName())) {
boolQuery.filter(
QueryBuilders.termQuery(HouseIndexKey.REGION_EN_NAME, rentSearch.getRegionEnName())
);
}
RentValueBlock area = RentValueBlock.matchArea(rentSearch.getAreaBlock());
if (!RentValueBlock.ALL.equals(area)) {
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(HouseIndexKey.AREA);
if (area.getMax() > 0) {
rangeQueryBuilder.lte(area.getMax());
}
if (area.getMin() > 0) {
rangeQueryBuilder.gte(area.getMin());
}
boolQuery.filter(rangeQueryBuilder);
}
RentValueBlock price = RentValueBlock.matchPrice(rentSearch.getPriceBlock());
if (!RentValueBlock.ALL.equals(price)) {
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(HouseIndexKey.PRICE);
if (price.getMax() > 0) {
rangeQuery.lte(price.getMax());
}
if (price.getMin() > 0) {
rangeQuery.gte(price.getMin());
}
boolQuery.filter(rangeQuery);
}
if (rentSearch.getDirection() > 0) {
boolQuery.filter(
QueryBuilders.termQuery(HouseIndexKey.DIRECTION, rentSearch.getDirection())
);
}
if (rentSearch.getRentWay() > -1) {
boolQuery.filter(
QueryBuilders.termQuery(HouseIndexKey.RENT_WAY, rentSearch.getRentWay())
);
}
// boolQuery.must(
// QueryBuilders.matchQuery(HouseIndexKey.TITLE, rentSearch.getKeywords())
// .boost(2.0f)
// );
boolQuery.must(
QueryBuilders.multiMatchQuery(rentSearch.getKeywords(),
HouseIndexKey.TITLE,
HouseIndexKey.TRAFFIC,
HouseIndexKey.DISTRICT,
HouseIndexKey.ROUND_SERVICE,
HouseIndexKey.SUBWAY_LINE_NAME,
HouseIndexKey.SUBWAY_STATION_NAME
));
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addSort(
HouseSort.getSortKey(rentSearch.getOrderBy()),
SortOrder.fromString(rentSearch.getOrderDirection())
)
.setFrom(rentSearch.getStart())
.setSize(rentSearch.getSize())
.setFetchSource(HouseIndexKey.HOUSE_ID, null);
logger.debug(requestBuilder.toString());
List houseIds = new ArrayList<>();
SearchResponse response = requestBuilder.get();
if (response.status() != RestStatus.OK) {
logger.warn("Search status is no ok for " + requestBuilder);
return new ServiceMultiResult<>(0, houseIds);
}
for (SearchHit hit : response.getHits()) {
System.out.println(hit.getSource());
houseIds.add(Longs.tryParse(String.valueOf(hit.getSource().get(HouseIndexKey.HOUSE_ID))));
}
return new ServiceMultiResult<>(response.getHits().totalHits, houseIds);
}
@Override
public ServiceResult> suggest(String prefix) {
CompletionSuggestionBuilder suggestion = SuggestBuilders.completionSuggestion("suggest").prefix(prefix).size(5);
SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.addSuggestion("autocomplete", suggestion);
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.suggest(suggestBuilder);
logger.debug(requestBuilder.toString());
SearchResponse response = requestBuilder.get();
Suggest suggest = response.getSuggest();
if (suggest == null) {
return ServiceResult.of(new ArrayList<>());
}
Suggest.Suggestion result = suggest.getSuggestion("autocomplete");
int maxSuggest = 0;
Set suggestSet = new HashSet<>();
for (Object term : result.getEntries()) {
if (term instanceof CompletionSuggestion.Entry) {
CompletionSuggestion.Entry item = (CompletionSuggestion.Entry) term;
if (item.getOptions().isEmpty()) {
continue;
}
for (CompletionSuggestion.Entry.Option option : item.getOptions()) {
String tip = option.getText().string();
if (suggestSet.contains(tip)) {
continue;
}
suggestSet.add(tip);
maxSuggest++;
}
}
if (maxSuggest > 5) {
break;
}
}
List suggests = Lists.newArrayList(suggestSet.toArray(new String[]{}));
return ServiceResult.of(suggests);
}
@Override
public ServiceResult aggregateDistrictHouse(String cityEnName, String regionEnName, String district) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
.filter(QueryBuilders.termQuery(HouseIndexKey.CITY_EN_NAME, cityEnName))
.filter(QueryBuilders.termQuery(HouseIndexKey.REGION_EN_NAME, regionEnName))
.filter(QueryBuilders.termQuery(HouseIndexKey.DISTRICT, district));
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addAggregation(
AggregationBuilders.terms(HouseIndexKey.AGG_DISTRICT)
.field(HouseIndexKey.DISTRICT)
).setSize(0);
logger.debug(requestBuilder.toString());
SearchResponse response = requestBuilder.get();
if (response.status() == RestStatus.OK) {
Terms terms = response.getAggregations().get(HouseIndexKey.AGG_DISTRICT);
if (terms.getBuckets() != null && !terms.getBuckets().isEmpty()) {
return ServiceResult.of(terms.getBucketByKey(district).getDocCount());
}
} else {
logger.warn("Failed to Aggregate for " + HouseIndexKey.AGG_DISTRICT);
}
return ServiceResult.of(0L);
}
@Override
public ServiceMultiResult mapAggregate(String cityEnName) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.filter(QueryBuilders.termQuery(HouseIndexKey.CITY_EN_NAME, cityEnName));
AggregationBuilder aggBuilder = AggregationBuilders.terms(HouseIndexKey.AGG_REGION)
.field(HouseIndexKey.REGION_EN_NAME);
SearchRequestBuilder requestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addAggregation(aggBuilder);
logger.debug(requestBuilder.toString());
SearchResponse response = requestBuilder.get();
List buckets = new ArrayList<>();
if (response.status() != RestStatus.OK) {
logger.warn("Aggregate status is not ok for " + requestBuilder);
return new ServiceMultiResult<>(0, buckets);
}
Terms terms = response.getAggregations().get(HouseIndexKey.AGG_REGION);
for (Terms.Bucket bucket : terms.getBuckets()) {
buckets.add(new HouseBucketDTO(bucket.getKeyAsString(), bucket.getDocCount()));
}
return new ServiceMultiResult<>(response.getHits().getTotalHits(), buckets);
}
@Override
public ServiceMultiResult mapQuery(String cityEnName, String orderBy,
String orderDirection,
int start,
int size) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.filter(QueryBuilders.termQuery(HouseIndexKey.CITY_EN_NAME, cityEnName));
SearchRequestBuilder searchRequestBuilder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addSort(HouseSort.getSortKey(orderBy), SortOrder.fromString(orderDirection))
.setFrom(start)
.setSize(size);
List houseIds = new ArrayList<>();
SearchResponse response = searchRequestBuilder.get();
if (response.status() != RestStatus.OK) {
logger.warn("Search status is not ok for " + searchRequestBuilder);
return new ServiceMultiResult<>(0, houseIds);
}
for (SearchHit hit : response.getHits()) {
houseIds.add(Longs.tryParse(String.valueOf(hit.getSource().get(HouseIndexKey.HOUSE_ID))));
}
return new ServiceMultiResult<>(response.getHits().getTotalHits(), houseIds);
}
@Override
public ServiceMultiResult mapQuery(MapSearch mapSearch) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.filter(QueryBuilders.termQuery(HouseIndexKey.CITY_EN_NAME, mapSearch.getCityEnName()));
boolQuery.filter(
QueryBuilders.geoBoundingBoxQuery("location")
.setCorners(
new GeoPoint(mapSearch.getLeftLatitude(), mapSearch.getLeftLongitude()),
new GeoPoint(mapSearch.getRightLatitude(), mapSearch.getRightLongitude())
));
SearchRequestBuilder builder = this.esClient.prepareSearch(INDEX_NAME)
.setTypes(INDEX_TYPE)
.setQuery(boolQuery)
.addSort(HouseSort.getSortKey(mapSearch.getOrderBy()),
SortOrder.fromString(mapSearch.getOrderDirection()))
.setFrom(mapSearch.getStart())
.setSize(mapSearch.getSize());
List houseIds = new ArrayList<>();
SearchResponse response = builder.get();
if (RestStatus.OK != response.status()) {
logger.warn("Search status is not ok for " + builder);
return new ServiceMultiResult<>(0, houseIds);
}
for (SearchHit hit : response.getHits()) {
houseIds.add(Longs.tryParse(String.valueOf(hit.getSource().get(HouseIndexKey.HOUSE_ID))));
}
return new ServiceMultiResult<>(response.getHits().getTotalHits(), houseIds);
}
private boolean updateSuggest(HouseIndexTemplate indexTemplate) {
AnalyzeRequestBuilder requestBuilder = new AnalyzeRequestBuilder(
this.esClient, AnalyzeAction.INSTANCE, INDEX_NAME, indexTemplate.getTitle(),
indexTemplate.getLayoutDesc(), indexTemplate.getRoundService(),
indexTemplate.getDescription(), indexTemplate.getSubwayLineName(),
indexTemplate.getSubwayStationName());
requestBuilder.setAnalyzer("ik_smart");
AnalyzeResponse response = requestBuilder.get();
List tokens = response.getTokens();
if (tokens == null) {
logger.warn("Can not analyze token for house: " + indexTemplate.getHouseId());
return false;
}
List suggests = new ArrayList<>();
for (AnalyzeResponse.AnalyzeToken token : tokens) {
// 排序数字类型 & 小于2个字符的分词结果
if ("".equals(token.getType()) || token.getTerm().length() < 2) {
continue;
}
HouseSuggest suggest = new HouseSuggest();
suggest.setInput(token.getTerm());
suggests.add(suggest);
}
// 定制化小区自动补全
HouseSuggest suggest = new HouseSuggest();
suggest.setInput(indexTemplate.getDistrict());
suggests.add(suggest);
indexTemplate.setSuggest(suggests);
return true;
}
private void remove(Long houseId, int retry) {
if (retry > HouseIndexMessage.MAX_RETRY) {
logger.error("Retry remove times over 3 for house: " + houseId + " Please check it!");
return;
}
HouseIndexMessage message = new HouseIndexMessage(houseId, HouseIndexMessage.REMOVE, retry);
try {
this.kafkaTemplate.send(INDEX_TOPIC, objectMapper.writeValueAsString(message));
} catch (JsonProcessingException e) {
logger.error("Cannot encode json for " + message, e);
}
}
}