ElasticsearchUtils操作

package com.tools.es.util;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.annotation.PostConstruct;

import org.apache.commons.collections.CollectionUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.reindex.BulkIndexByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.index.reindex.DeleteByQueryRequestBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.range.Range;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.api.dto.SamplingResultVO;
import com.server.jdbc.bean.IndexTableEntity;
import com.server.tools.es.dataware.BaseInfo;
import com.server.tools.es.dataware.ErrorDetail;
import com.server.tools.es.dataware.Filter;
import com.server.tools.es.dataware.FilterDetail;
import com.server.tools.es.dataware.Query;
import com.server.tools.es.dataware.QueryColumn;
import com.server.tools.es.dataware.QueryOrder;
import com.server.tools.es.dataware.ResultMsg;
import com.server.tools.es.dto.Column;
import com.server.tools.es.dto.DataSet;
import com.server.tools.es.enumeration.EnumExtendDate;
import com.server.tools.es.type.ColType;

/**
 * ES 工具类
 * 
 * @author duhai
 * @version 1.0
 */
@Component
public class ElasticsearchUtils {

	private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchUtils.class);

	@Autowired
	private TransportClient transportClient;

	private static TransportClient client;

	@PostConstruct
	public void init() {
		client = this.transportClient;
	}

	private final static String TIMESTAMP = "timestamp";

	public static final String FLAG_DIMENSION = "dimention_";
	public static final String FLAG_COMPARSION = "comparsion_";
	public static final String FLAG_MEASURE = "measure_";

	public static final String FLAG_ENDFIX_KEYWORD = "";

	public static final String FLAG_ENDFIX_DOUBLE_STRING = "_dstr";
	public static final String FLAG_ENDFIX_TEXT_ANALYSE = "word";

	@Value("${sampling.index}")
	private String indexName;

	public static final int BULK_SIZE = 10000;

	/**
	 * 创建索引
	 * 
	 * @param columnList
	 * @param indexName
	 */
	public void createPhyTable(final List columnList, final String indexName, final boolean flag) {
		if (!existTable(indexName)) {
			Settings esSetting = Settings.builder().put("index.refresh_interval", "30s")
					// 索引刷新间隔调整
					// .put("index.translog.durability", "async")
					// //如果对数据丢失有一定的容忍，可以打开async模式
					.put("index.translog.sync_interval", "30s").put("index.translog.flush_threshold_size", "1gb")
					.put("index.merge.scheduler.max_thread_count", 1).build();
			// 创建索引
			client.admin().indices().prepareCreate(indexName).setSettings(esSetting).execute().actionGet();

		}

		// 创建索引结构
		XContentBuilder builder = null;
		try {
			builder = XContentFactory.jsonBuilder().startObject().startObject(indexName).startObject("_all")
					.field("enabled", false).endObject()
					// .startObject("_source").field("enabled",false).endObject()
					.startObject("properties");
			for (Column colume : columnList) {
				builder.startObject(colume.getColName());

				if (ColType.TEXT == colume.getColType().intValue()) {
					builder.field("type", "string");
					builder.field("index", "not_analyzed").endObject();

					/*
					 * builder.startObject("fields"); builder.startObject(FLAG_ENDFIX_TEXT_ANALYSE);
					 * builder.field("type", "text"); builder.field("fielddata", true); builder.field("analyzer",
					 * "not_analyzed"); builder.field("search_analyzer", "not_analyzed"); builder.field("index",
					 * "analyzed").endObject().endObject();
					 */
				} else if (ColType.NUMBER == colume.getColType().intValue()) {
					builder.field("type", "double");
					builder.field("index", "not_analyzed").endObject();

					if (flag) {
						builder.startObject(colume.getColName() + FLAG_ENDFIX_DOUBLE_STRING);
						builder.field("type", "string");
						builder.field("index", "not_analyzed").endObject();
					}

				} else if (ColType.DATE == colume.getColType().intValue()) {
					builder.field("type", "date").field("format", "yyyy/MM/dd||yyyy/MM/dd HH:mm:ss||epoch_millis");
					builder.field("index", "not_analyzed").endObject();
				} /*
					 * else if (ColType.TEXT_ANALYSE == colume.getColType().intValue()) { builder.startObject("fields");
					 * builder.startObject(FLAG_ENDFIX_TEXT_ANALYSE); builder.field("type", "text");
					 * builder.field("fielddata", true); builder.field("analyzer", "not_analyzed");
					 * builder.field("search_analyzer", "not_analyzed"); builder.field("index",
					 * "analyzed").endObject().endObject().endObject(); }
					 */
			}

			builder.endObject().endObject().endObject();
		} catch (IOException e) {
			LOGGER.error("创建索引结构失败: " + indexName);
			throw new RuntimeException("创建索引结构失败: " + indexName, e);
		}

		PutMappingRequest mapping = Requests.putMappingRequest(indexName).type(indexName).source(builder);
		client.admin().indices().putMapping(mapping).actionGet();

		// client.close();

		LOGGER.info("Create Index Structure Succeed：" + indexName);
	}

	/**
	 * 创建索引2
	 * 
	 * @param columnList
	 * @param indexName
	 */
	public void createPhyTable2(final List columnList, final String indexName, final boolean flag) {
		if (!existTable(indexName)) {
			Settings esSetting = Settings.builder().put("index.refresh_interval", "30s")
					// 索引刷新间隔调整
					// .put("index.translog.durability", "async")
					// //如果对数据丢失有一定的容忍，可以打开async模式
					.put("index.translog.sync_interval", "30s").put("index.translog.flush_threshold_size", "1gb")
					.put("index.merge.scheduler.max_thread_count", 1).build();
			// 创建索引
			client.admin().indices().prepareCreate(indexName).setSettings(esSetting).execute().actionGet();

		}

		// 创建索引结构
		XContentBuilder builder = null;
		try {
			builder = XContentFactory.jsonBuilder().startObject().startObject(indexName).startObject("_all")
					.field("enabled", false).endObject()
					// .startObject("_source").field("enabled",false).endObject()
					.startObject("properties");
			for (Column colume : columnList) {
				builder.startObject(colume.getColName());

				if (ColType.TEXT == colume.getColType().intValue()) {
					builder.field("type", "string");
					builder.field("index", "not_analyzed").endObject();

					/*
					 * builder.startObject("fields"); builder.startObject(FLAG_ENDFIX_TEXT_ANALYSE);
					 * builder.field("type", "text"); builder.field("fielddata", true); builder.field("analyzer",
					 * "not_analyzed"); builder.field("search_analyzer", "not_analyzed"); builder.field("index",
					 * "analyzed").endObject().endObject();
					 */
				} else if (ColType.NUMBER == colume.getColType().intValue()) {
					builder.field("type", "double");
					builder.field("index", "not_analyzed").endObject();

					if (flag) {
						builder.startObject(colume.getColName() + FLAG_ENDFIX_DOUBLE_STRING);
						builder.field("type", "string");
						builder.field("index", "not_analyzed").endObject();
					}

				} else if (ColType.DATE == colume.getColType().intValue()) {
					builder.field("type", "date").field("format", "yyyy/MM/dd||yyyy/MM/dd HH:mm:ss||epoch_millis");
					builder.field("index", "not_analyzed").endObject();
				} /*
					 * else if (ColType.TEXT_ANALYSE == colume.getColType().intValue()) { builder.startObject("fields");
					 * builder.startObject(FLAG_ENDFIX_TEXT_ANALYSE); builder.field("type", "text");
					 * builder.field("fielddata", true); builder.field("analyzer", "not_analyzed");
					 * builder.field("search_analyzer", "not_analyzed"); builder.field("index",
					 * "analyzed").endObject().endObject().endObject(); }
					 */
			}

			builder.endObject().endObject().endObject();
		} catch (IOException e) {
			LOGGER.error("创建索引结构失败: " + indexName);
			throw new RuntimeException("创建索引结构失败: " + indexName, e);
		}

		PutMappingRequest mapping = Requests.putMappingRequest(indexName).type(indexName).source(builder);
		client.admin().indices().putMapping(mapping).actionGet();

		// client.close();

		LOGGER.info("Create Index Structure Succeed：" + indexName);
	}

	/**
	 * 判断索引是否存在
	 * 
	 * @param table
	 * @return
	 */
	public boolean existTable(final String table) {
		IndicesExistsRequest request = new IndicesExistsRequest(table);
		IndicesExistsResponse response = client.admin().indices().exists(request).actionGet();
		if (response.isExists()) {
			return true;
		}
		return false;
	}

	/**
	 * 写入历史数据
	 * 
	 * @param resultList
	 *            voList
	 * @return ResultMsg 结果
	 */
	public ResultMsg saveSampling(final List resultList) {
		final ResultMsg resultMsg = new ResultMsg();
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		int count = 0;
		int len = resultList.size();
		try {
			for (int i = 0; i < len; i++) {
				final SamplingResultVO vo = resultList.get(i);
				final XContentBuilder builder = XContentFactory.jsonBuilder().startObject();

				final String id = vo.getId();

				builder.field("timestamp", vo.getTimestamp());
				builder.field("deviceId", vo.getDeviceId());
				builder.field("metric", vo.getMetric());
				builder.field("indexName", vo.getIndexName());
				builder.field("domain", vo.getDomain());
				builder.field("rule", vo.getRule());
				builder.field("top", vo.getTop());
				builder.field("height", vo.getHeight());
				builder.field("flat", vo.getFlat());
				builder.field("valley", vo.getValley());
				builder.field("topDj", vo.getTopDj());
				builder.field("heightDj", vo.getHeightDj());
				builder.field("flatDj", vo.getFlatDj());
				builder.field("valleyDj", vo.getValleyDj());

				builder.endObject();

				IndexRequestBuilder requestBuilder = client.prepareIndex("sampling", indexName, id).setSource(builder);
				bulkRequest.add(requestBuilder);
				count++;

				if (count % BULK_SIZE == 0) {
					BulkResponse bulkResponse = bulkRequest.execute().actionGet();
					if (bulkResponse.hasFailures()) {
						LOGGER.error("导入索引数据失败: " + indexName);
						LOGGER.error("导入索引数据失败: " + bulkResponse.buildFailureMessage());

						resultMsg.setHasFailures(true);

						BulkItemResponse[] responses = bulkResponse.getItems();
						for (int k = 0; k < responses.length; k++) {
							BulkItemResponse response = responses[k];
							if (response.isFailed()) {
								ErrorDetail errorDetail = new ErrorDetail();
								errorDetail.setIndex(k + (i / BULK_SIZE));
								errorDetail.setId(response.getId());
								errorDetail.setMsg(response.getFailureMessage());

								resultMsg.addError(errorDetail);
							}
						}
					}

					bulkRequest = client.prepareBulk();
					bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
					count = 0;
				}
			}

			if (count > 0) {
				BulkResponse bulkResponse = bulkRequest.execute().actionGet();
				if (bulkResponse.hasFailures()) {
					LOGGER.error("导入索引数据失败: " + indexName);
					LOGGER.error("导入索引数据失败: " + bulkResponse.buildFailureMessage());

					resultMsg.setHasFailures(true);

					BulkItemResponse[] responses = bulkResponse.getItems();
					for (int k = 0; k < responses.length; k++) {
						BulkItemResponse response = responses[k];
						if (response.isFailed()) {
							ErrorDetail errorDetail = new ErrorDetail();
							errorDetail.setIndex(k + (len / BULK_SIZE));
							errorDetail.setId(response.getId());
							errorDetail.setMsg(response.getFailureMessage());

							resultMsg.addError(errorDetail);
						}
					}
				}
			}

			return resultMsg;
		} catch (Exception e) {
			LOGGER.error("导入索引数据失败: " + indexName);
			throw new RuntimeException("导入索引数据失败: " + indexName, e);
		} finally {
			bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
		}

	}

	/**
	 * 数据处理(写入)
	 * 
	 * @param operator
	 * @param dataSet
	 * @return
	 */
	public ResultMsg dealDataHelp(final DataSet dataSet) {
		String indexName = dataSet.getPhyName();
		List columnList = dataSet.getCols();

		List> dataList = dataSet.getDatas();

		List> datas = new ArrayList>();
		for (Map map : dataList) {
			List row = new ArrayList();
			for (Column col : columnList) {
				Object val = map.get(col.getColName());
				if (val != null)
					row.add(val + "");
				else
					row.add(null);
			}

			datas.add(row);
		}

		ResultMsg resultMsg = null;
		resultMsg = dealDataHelp(indexName, columnList, datas);

		return resultMsg;
	}

	/**
	 * 数据写入
	 * 
	 * @param indexName
	 * @param columnList
	 * @param dataList
	 * @return
	 */
	private ResultMsg dealDataHelp(final String indexName, final List columnList,
			final List> dataList) {
		ResultMsg resultMsg = new ResultMsg();

		BulkRequestBuilder bulkRequest = client.prepareBulk();
		bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
		int count = 0;
		int len = dataList.size();
		try {
			for (int i = 0; i < len; i++) {
				List rowData = dataList.get(i);
				XContentBuilder builder = XContentFactory.jsonBuilder().startObject();

				String id = null;

				for (int j = 0; j < columnList.size(); j++) {
					Column column = columnList.get(j);
					String val = "";
					try {
						val = rowData.get(j);
					} catch (IndexOutOfBoundsException e) {
						// e.printStackTrace();
						if (ColType.NUMBER.equals(column.getColType())) {
							val = "0";
						} else {
							val = "";
						}
					}

					// 尝试去除千分符
					if (val != null && ColType.NUMBER.equals(column.getColType())) {
						val = val.replace(",", "");
					}

					// 日期类型空串设置为null
					if (ColType.DATE.equals(column.getColType()) && "".equals(val)) {
						val = null;
					}

					builder.field(column.getColName(), val);

					/*
					 * if(ColType.NUMBER.equals(column.getColType())){ builder.field(column.getColName()+
					 * FLAG_ENDFIX_DOUBLE_STRING,val); }
					 */

				}

				builder.endObject();

				IndexRequestBuilder requestBuilder = client.prepareIndex(indexName, indexName, id).setSource(builder);
				bulkRequest.add(requestBuilder);

				count++;

				if (count % BULK_SIZE == 0) {
					BulkResponse bulkResponse = bulkRequest.execute().actionGet();
					if (bulkResponse.hasFailures()) {
						LOGGER.error("导入索引数据失败: " + indexName);
						LOGGER.error("导入索引数据失败: " + bulkResponse.buildFailureMessage());

						resultMsg.setHasFailures(true);

						BulkItemResponse[] responses = bulkResponse.getItems();
						for (int k = 0; k < responses.length; k++) {
							BulkItemResponse response = responses[k];
							if (response.isFailed()) {
								ErrorDetail errorDetail = new ErrorDetail();
								errorDetail.setIndex(k + (i / BULK_SIZE));
								errorDetail.setId(response.getId());
								errorDetail.setMsg(response.getFailureMessage());

								resultMsg.addError(errorDetail);
							}
						}
					}

					bulkRequest = client.prepareBulk();
					bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
					count = 0;
				}
			}

			if (count > 0) {
				BulkResponse bulkResponse = bulkRequest.execute().actionGet();
				if (bulkResponse.hasFailures()) {
					LOGGER.error("导入索引数据失败: " + indexName);
					LOGGER.error("导入索引数据失败: " + bulkResponse.buildFailureMessage());
					// throw new RuntimeException("导入索引数据失败: "+indexName);

					resultMsg.setHasFailures(true);

					BulkItemResponse[] responses = bulkResponse.getItems();
					for (int k = 0; k < responses.length; k++) {
						BulkItemResponse response = responses[k];
						if (response.isFailed()) {
							ErrorDetail errorDetail = new ErrorDetail();
							errorDetail.setIndex(k + (len / BULK_SIZE));
							errorDetail.setId(response.getId());
							errorDetail.setMsg(response.getFailureMessage());

							resultMsg.addError(errorDetail);
						}
					}
				}
			}

			return resultMsg;
		} catch (IOException e) {
			LOGGER.error("导入索引数据失败: " + indexName);
			throw new RuntimeException("导入索引数据失败: " + indexName, e);
		} finally {
			bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
		}
	}

	/**
	 * 数据写入
	 * 
	 * @param indexName
	 * @param columnList
	 * @param dataList
	 * @return
	 */
	public ResultMsg dealDataHelpV1(final String indexName, final List columnList, final JSONArray dataList) {
		ResultMsg resultMsg = new ResultMsg();

		BulkRequestBuilder bulkRequest = client.prepareBulk();
		bulkRequest.setRefreshPolicy(RefreshPolicy.NONE);
		int count = 0;
		int len = dataList.size();
		try {
			for (int i = 0; i < len; i++) {
				JSONObject rowData = dataList.getJSONObject(i);
				XContentBuilder builder = XContentFactory.jsonBuilder().startObject();

				for (int j = 0; j < columnList.size(); j++) {
					Column column = columnList.get(j);
					String val = "";
					try {
						val = rowData.getString(column.getColName());
					} catch (IndexOutOfBoundsException e) {
						if (ColType.NUMBER.equals(column.getColType())) {
							val = "0";
						} else {
							val = "";
						}
					}

					// 尝试去除千分符
					if (val != null && ColType.NUMBER.equals(column.getColType())) {
						val = val.replace(",", "");
					}

					// 日期类型空串设置为null
					if (ColType.DATE.equals(column.getColType()) && "".equals(val)) {
						val = null;
					}

					builder.field(column.getColName(), val);

				}

				builder.endObject();

				IndexRequestBuilder requestBuilder = client.prepareIndex(indexName, indexName).setSource(builder);
				bulkRequest.add(requestBuilder);

				count++;

				if (count % BULK_SIZE == 0) {
					long s1 = System.currentTimeMillis();
					BulkResponse bulkResponse = bulkRequest.execute().actionGet();
					LOGGER.info("##### 调用ES BULK 批量插入{}条数据结束 ##### 时间: {}", BULK_SIZE, System.currentTimeMillis() - s1);
					if (bulkResponse.hasFailures()) {
						LOGGER.error("导入索引数据失败: " + indexName);
						LOGGER.error("导入索引数据失败: " + bulkResponse.buildFailureMessage());

						resultMsg.setHasFailures(true);

						BulkItemResponse[] responses = bulkResponse.getItems();
						for (int k = 0; k < responses.length; k++) {
							BulkItemResponse response = responses[k];
							if (response.isFailed()) {
								ErrorDetail errorDetail = new ErrorDetail();
								errorDetail.setIndex(k + (i / BULK_SIZE));
								errorDetail.setId(response.getId());
								errorDetail.setMsg(response.getFailureMessage());

								resultMsg.addError(errorDetail);
							}
						}
					}

					bulkRequest = client.prepareBulk();
					bulkRequest.setRefreshPolicy(RefreshPolicy.NONE);
					count = 0;
				}
			}

			if (count > 0) {
				long endTime = System.currentTimeMillis();
				BulkResponse bulkResponse = bulkRequest.execute().actionGet();
				LOGGER.info("##### 调用ES BULK 批量插入{}条数据结束 ##### 时间: {}", count, System.currentTimeMillis() - endTime);
				if (bulkResponse.hasFailures()) {
					LOGGER.error("导入索引数据失败: " + indexName);
					LOGGER.error("导入索引数据失败: " + bulkResponse.buildFailureMessage());
					// throw new RuntimeException("导入索引数据失败: "+indexName);

					resultMsg.setHasFailures(true);

					BulkItemResponse[] responses = bulkResponse.getItems();
					for (int k = 0; k < responses.length; k++) {
						BulkItemResponse response = responses[k];
						if (response.isFailed()) {
							ErrorDetail errorDetail = new ErrorDetail();
							errorDetail.setIndex(k + (len / BULK_SIZE));
							errorDetail.setId(response.getId());
							errorDetail.setMsg(response.getFailureMessage());

							resultMsg.addError(errorDetail);
						}
					}
				}
			}

			return resultMsg;
		} catch (IOException e) {
			LOGGER.error("导入索引数据失败: " + indexName);
			throw new RuntimeException("导入索引数据失败: " + indexName, e);
		} finally {
			bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
		}
	}

	/**
	 * 删除索引
	 * 
	 * @param phyTableName
	 */
	public void dropPhyTable(final String indexName) {
		try {
			DeleteIndexResponse dResponse = client.admin().indices().prepareDelete(indexName).execute().actionGet();
			if (dResponse.isAcknowledged()) {
				LOGGER.info("delete index " + indexName + "  successfully!");
			} else {
				String msg = "dropPhyTable(): Fail to delete index " + indexName;
				LOGGER.error(msg);
				// throw new RuntimeException(msg);
			}
		} catch (Exception e) {
			String msg = "dropPhyTable(): Fail to delete index " + indexName;
			LOGGER.error(msg);
		}

	}

	/**
	 * 数据查询（查询）
	 */
	private static final int startRow = 0;
	private static final int pageSize = 200;
	private static final String FLAG_FIRST_SCOLL = "FLAG_FIRST_SCOLL";

	public DataSet dealData(final JSONObject json, final List indexList) {
		json.put("index", indexList.get(0).getIndex());
		JSONObject colsjson = new JSONObject();
		JSONArray timejson = json.getJSONArray("term");
		for (IndexTableEntity index : indexList) {
			colsjson.put(index.getFieldName(), index.getFieldType());

			for (int i = 0; i < timejson.size(); i++) {
				String fieldName = timejson.getJSONObject(i).getString("term");
				if (fieldName.equals(index.getFieldName())) {
					timejson.getJSONObject(i).put("type", index.getFieldType());
				}
			}
		}

		Query query = new Query();
		query.setPhyTableName(json.getString("index"));
		if (!json.getJSONObject("page").isEmpty()) {
			query.setStartRow(Integer.valueOf(json.getJSONObject("page").getString("start")));
			query.setPageSize(Integer.valueOf(json.getJSONObject("page").getString("end")));
		} else {
			query.setStartRow(startRow);
			query.setPageSize(pageSize);// Query.pageSize 最大值取10000,超过此值会出问题
		}

		// 待查询的属性
		List cols = BaseInfo.getQueryCols(colsjson);
		query.setColumnList(cols);

		// 多个过滤条件之前，是“与”的关系
		query.setConnFlag(Filter.CONN_FLAG_AND);
		query.setFilterList(BaseInfo.getFilterList(timejson.toJavaList(JSONObject.class)));

		// 时间排列顺序
		QueryOrder queryOrder = new QueryOrder();
		queryOrder.setField(json.getJSONObject("order").getString("term"));
		queryOrder.setOrder(json.getJSONObject("order").getString("order"));
		query.addOrder(queryOrder);

		DataSet dataSet = commonQuery(query);

		return dataSet;
	}

	/**
	 * 普通查询
	 * 
	 * @param indexName
	 * @param query
	 * @return
	 */

	private DataSet commonQuery(final Query query) {
		String phyIndexName = query.getPhyTableName();
		List columnList = query.getColumnList();

		SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
		// 如果开启游标，则滚动获取
		if (!FLAG_FIRST_SCOLL.equals(query.getScollId())) {
			sourceBuilder.from(query.getStartRow()).size(query.getPageSize());
		} else {
			sourceBuilder.size(startRow);
		}

		List filterList = query.getFilterList();
		if (filterList != null && filterList.size() > 0) {
			BoolQueryBuilder mainQuery = QueryBuilders.boolQuery();
			for (Filter filter : filterList) {
				List queryBuilders = dealFilter(filter);
				if (CollectionUtils.isEmpty(queryBuilders)) {
					continue;
				}
				for (QueryBuilder queryBuilder : queryBuilders) {
					if (query.getConnFlag().equals(Filter.CONN_FLAG_OR)) {
						mainQuery.should(queryBuilder);
					} else {
						mainQuery.must(queryBuilder);
					}
				}

			}
			sourceBuilder.query(mainQuery);
		}

		for (QueryColumn column : columnList) {
			if (QueryColumn.COL_TYPE_COMPUTE.equals(column.getColType())) {
				sourceBuilder.scriptField(column.getPhyColName() + FLAG_ENDFIX_KEYWORD, column.getScript(), true);
			} else {
				sourceBuilder.docValueField(column.getPhyColName() + FLAG_ENDFIX_KEYWORD);
			}
		}

		List orders = query.getOrders();
		if (orders != null && orders.size() > 0) {
			for (QueryOrder queryOrder : orders) {
				sourceBuilder.sort(queryOrder.getField(), SortOrder.fromString(queryOrder.getOrder()));
			}
		}
		LOGGER.debug("GET " + phyIndexName + "/_search \n" + sourceBuilder.toString());
		/*
		 * SearchRequest request = Requests.searchRequest(phyIndexName); request.source(sourceBuilder);
		 * 
		 * //如果开启游标，则滚动获取 if(Query.FLAG_FIRST_SCOLL.equals(query.getScollId())){ request.scroll("2m");
		 * //request.scroll(new TimeValue(30000)); }
		 * 
		 * //如果开启版本 if(query.isEnableVersion()){
		 * 
		 * }
		 * 
		 * SearchResponse response = client.search(request).actionGet();
		 */
		SearchRequestBuilder request = client.prepareSearch(phyIndexName).setSource(sourceBuilder);
		System.out.println(request.toString());

		// 如果开启游标，则滚动获取
		if (Query.FLAG_FIRST_SCOLL.equals(query.getScollId())) {
			request.setScroll("2m");
		}

		// 如果开启版本
		if (query.isEnableVersion()) {
			request.setVersion(true);
		}

		SearchResponse response = request.get();

		// 返回数据集
		DataSet dataSet = buildDataSet(query.isEnableVersion(), response, columnList);

		List cols = new ArrayList();
		for (QueryColumn column : columnList) {
			Column col = new Column();
			col.setColName(column.getPhyColName());
			col.setShowName(column.getAlias());
			col.setColType(column.getColDataType());
			cols.add(col);
		}

		dataSet.setCols(cols);
		dataSet.setScollId(response.getScrollId());
		return dataSet;
	}

	@SuppressWarnings("unused")
	private List dealFilter(final Filter filter) {
		List queryBuilders = new ArrayList();
		List detailList = filter.getDetailList();

		// if(detailList == null || detailList.size() == 0)
		// return queryBuilders;

		if (filter.getScript() != null) {
			queryBuilders.add(QueryBuilders.scriptQuery(filter.getScript()));
			return queryBuilders;
		}

		if (detailList != null) {
			for (FilterDetail detail : detailList) {
				String operator = detail.getOperator();
				if ("NOT_NULL".equals(operator) || "NULL".equals(operator)) {
					filter.setExtend("");
				}
			}
		}

		if (!StringUtils.isEmpty(filter.getExtend())) {

			// 当为时间筛选时，extend为year、month等
			String extend = filter.getExtend();
			EnumExtendDate extendDate = EnumExtendDate.getEnumType(extend);
			if (extendDate == null) {
				return queryBuilders;
			}
			for (FilterDetail detail : detailList) {
				BoolQueryBuilder detailQueryBuilder = QueryBuilders.boolQuery();
				String val = detail.getFilterVal();
				String operator = detail.getOperator();
				DateTime lowDate = DateUtils.paseTime(val, extendDate.getFormat());
				DateTime highDate = DateUtils.paseTime(val, extendDate.getFormat());
				switch (extendDate) {
				case YEAR:
					highDate = highDate.plusYears(1);
					break;
				case QUARTER:
					highDate = highDate.plusMonths(3);
					break;
				case MONTH:
					highDate = highDate.plusMonths(1);
					break;
				case WEEK:
					highDate = highDate.plusWeeks(1);
					break;
				case DAY:
					highDate = highDate.plusDays(1);
					break;
				case HOUR:
					highDate = highDate.plusHours(1);
					break;
				case MINUTE:
					highDate = highDate.plusMinutes(1);
					break;
				case SECOND:
					highDate = highDate.plusSeconds(1);
					break;
				default:
					break;
				}
				QueryBuilder childBuilder = null;
				childBuilder = QueryBuilders.rangeQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD)
						.from(DateUtils.formateDate(lowDate.toDate())).includeLower(true)
						.to(DateUtils.formateDate(highDate.toDate())).includeUpper(false);
				if ("NOT_EQUAL".equals(operator) || "NOT_CONTAINS".equals(operator)) {
					detailQueryBuilder.mustNot(childBuilder);
				} else {
					detailQueryBuilder.must(childBuilder);
				}
				queryBuilders.add(detailQueryBuilder);
			}
			return queryBuilders;
		}

		if (Filter.FILTER_TYPE_CLUED.equals(filter.getFilterType())) {
			List valList = new ArrayList();
			for (FilterDetail detail : detailList) {
				valList.add(detail.getFilterVal());
			}
			queryBuilders.add(QueryBuilders.termsQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, valList));
			return queryBuilders;
		} else if (Filter.FILTER_TYPE_CONDITION.equals(filter.getFilterType())) {

			for (int i = 0; i < detailList.size(); i++) {
				BoolQueryBuilder detailQueryBuilder = QueryBuilders.boolQuery();
				QueryBuilder childBuilder = QueryBuilders.rangeQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD);

				// 对于日期类型，把两个detail作为一组
				int k = 1;
				if (ColType.DATE.equals(filter.getDataType()) && (i + 2) <= detailList.size()) {
					k = 2;
				}

				String operator = null;
				String val = null;
				for (int j = 0; j < k; j++) {
					i = i + j;
					FilterDetail detail = detailList.get(i);

					operator = detail.getOperator();
					val = detail.getFilterVal();

					childBuilder = dealFilterDetailChild(filter, detail, childBuilder);

					if (FilterDetail.OPERATOR_NULL.equals(operator)
							|| FilterDetail.OPERATOR_NOT_NULL.equals(operator)) {
						setNullOperator(filter, detailQueryBuilder, operator);
						queryBuilders.add(detailQueryBuilder);
						childBuilder = null;
					}
				}
				if (childBuilder == null) {
					continue;
				}

				if (FilterDetail.OPERATOR_NOT_CONTAINS.equals(operator)
						|| FilterDetail.OPERATOR_NOT_EQUAL.equals(operator)) {
					detailQueryBuilder.mustNot(childBuilder);
				} else {
					detailQueryBuilder.must(childBuilder);
				}
				queryBuilders.add(detailQueryBuilder);
			}

			return queryBuilders;
		} else if (Filter.FILTER_TYPE_EXPRESSION.equals(filter.getFilterType())) {
			// TODO: 表达式解析
		} else if (Filter.FILTER_TYPE_QUERYSTRING.equals(filter.getFilterType())) {
			String phyColNamesStr = filter.getPhyColName();
			// 多个字段用","切分
			String[] phyColNames = phyColNamesStr.split(",");

			QueryStringQueryBuilder queryStringQueryBuilder = QueryBuilders.queryStringQuery(filter.getFilterValue());
			for (String string : phyColNames) {
				queryStringQueryBuilder = queryStringQueryBuilder.field(string + ".word");
			}
			queryBuilders.add(queryStringQueryBuilder);
			return queryBuilders;
		} else if (Filter.FILTER_TYPE_RANGER.equals(filter.getFilterType())) {
			if (detailList.size() > 0) {
				BoolQueryBuilder detailQueryBuilder = QueryBuilders.boolQuery();
				QueryBuilder childBuilder = QueryBuilders.rangeQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD);
				for (int i = 0; i < detailList.size(); i++) {
					FilterDetail detail = detailList.get(i);
					childBuilder = dealFilterDetailChild(filter, detail, childBuilder);
				}
				detailQueryBuilder.must(childBuilder);
				queryBuilders.add(detailQueryBuilder);
			}
		}
		return queryBuilders;
	}

	private QueryBuilder dealFilterDetailChild(final Filter filter, final FilterDetail detail,
			QueryBuilder childBuilder) {
		String operator = detail.getOperator();
		String val = detail.getFilterVal();

		if (StringUtils.isEmpty(val)) {
			if (!(FilterDetail.OPERATOR_NULL.equals(operator) || FilterDetail.OPERATOR_NOT_NULL.equals(operator))) {
				return childBuilder;
			}
		}
		if (FilterDetail.OPERATOR_START_WITH.equals(operator)) {
			childBuilder = QueryBuilders.prefixQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, val);
		} else if (FilterDetail.OPERATOR_END_WITH.equals(operator)) {
			childBuilder = QueryBuilders.wildcardQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, "*" + val);
		} else if (FilterDetail.OPERATOR_CONTAINS.equals(operator)) {
			childBuilder = QueryBuilders.wildcardQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, "*" + val + "*");
		} else if (FilterDetail.OPERATOR_NOT_CONTAINS.equals(operator)) {
			childBuilder = QueryBuilders.wildcardQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, "*" + val + "*");
		} else if (FilterDetail.OPERATOR_EQUAL.equals(operator) || FilterDetail.OPERATOR_NOT_EQUAL.equals(operator)) {
			childBuilder = QueryBuilders.matchQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, val);
		} else if (FilterDetail.OPERATOR_MORE.equals(operator)) {
			childBuilder = ((RangeQueryBuilder) childBuilder).from(val).includeLower(false);
		} else if (FilterDetail.OPERATOR_MORE_AND_EQUAL.equals(operator)) {
			childBuilder = ((RangeQueryBuilder) childBuilder).from(val).includeLower(true);
		} else if (FilterDetail.OPERATOR_LESS.equals(operator)) {
			childBuilder = ((RangeQueryBuilder) childBuilder).to(val).includeUpper(false);
		} else if (FilterDetail.OPERATOR_LESS_AND_EQUAL.equals(operator)) {
			childBuilder = ((RangeQueryBuilder) childBuilder).to(val).includeUpper(true);
		}

		return childBuilder;
	}

	private void setNullOperator(final Filter filter, final BoolQueryBuilder detailQueryBuilder,
			final String operator) {
		// 对于日期和文本的为空处理，采用existQuery查询，所以相反
		BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
		QueryBuilder newchildBuilder = null;
		if (filter.getDataType().equals(ColType.TEXT)) {
			newchildBuilder = QueryBuilders.matchQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, "");
		} else if (filter.getDataType().equals(ColType.NUMBER)) {
			newchildBuilder = QueryBuilders.matchQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD, "0");
		}

		if (newchildBuilder != null) {
			if (FilterDetail.OPERATOR_NOT_NULL.equals(operator)) {
				boolQueryBuilder.mustNot(newchildBuilder);
				detailQueryBuilder.must(boolQueryBuilder);
			} else {
				boolQueryBuilder.must(newchildBuilder);
				detailQueryBuilder.should(boolQueryBuilder);
			}
		}

		QueryBuilder childBuilder1 = QueryBuilders.existsQuery(filter.getPhyColName() + FLAG_ENDFIX_KEYWORD);
		BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
		if (FilterDetail.OPERATOR_NULL.equals(operator)) {
			boolQueryBuilder1.mustNot(childBuilder1);
			detailQueryBuilder.should(boolQueryBuilder1);
		} else if (FilterDetail.OPERATOR_NOT_NULL.equals(operator)) {
			boolQueryBuilder1.must(childBuilder1);
			detailQueryBuilder.must(boolQueryBuilder1);
		}
	}

	private DataSet buildDataSet(final boolean useVersion, final SearchResponse response,
			final List columnList) {
		// 返回数据集
		DataSet dataSet = new DataSet();
		SearchHits searchHits = response.getHits();
		dataSet.setTotal(searchHits.getTotalHits());
		SearchHit[] hits = searchHits.getHits();
		List> datas = new ArrayList>();
		List data = new ArrayList();
		for (int i = 0; i < hits.length; i++) {
			SearchHit hit = hits[i];

			// Map recordMap = hit.getSource();
			Map fields = hit.getFields();
			Map recordMap = new HashMap();
			JSONObject recordJSON = new JSONObject();

			for (QueryColumn column : columnList) {
				SearchHitField object = fields.get(column.getPhyColName() + FLAG_ENDFIX_KEYWORD);
				if (object != null) {
					Object value = object.getValue();
					if (value == null) {
						value = "";
					}
					if (column.getColDataType().equals(ColType.DATE)) {
						try {
							if (value instanceof Date) {
								recordMap.put(column.getPhyColName(), DateUtils.formateDate((Date) value));
								recordJSON.put(column.getPhyColName(), DateUtils.formateDate((Date) value));
							} else {
								recordMap.put(column.getPhyColName(), DateUtils.paseGMTTime((long) value));
								recordJSON.put(column.getPhyColName(), DateUtils.paseGMTTime((long) value));
							}
						} catch (Exception e) {
							recordMap.put(column.getPhyColName(), value);
							recordJSON.put(column.getPhyColName(), value);
						}
					} else {
						recordMap.put(column.getPhyColName(), value);
						recordJSON.put(column.getPhyColName(), value);
					}
				} else {
					// null前台显示不了
					recordMap.put(column.getPhyColName(), "");
					recordJSON.put(column.getPhyColName(), "");
				}
			}

			// 如果开启了版本
			if (useVersion) {
				recordMap.put("_version", hit.getVersion());
				recordJSON.put("_version", hit.getVersion());
			}
			datas.add(recordMap);
			data.add(recordJSON);
		}

		dataSet.setUseVersion(useVersion);
		dataSet.setDatas(datas);
		dataSet.setData(data);
		return dataSet;
	}

	/**
	 * 聚合查询
	 * 
	 * @param client
	 * @param json
	 * @param indexList
	 * @return
	 */
	public JSONArray queryGraph(final JSONObject json, final String indexName) {

		JSONArray jsonlist = new JSONArray();
		JSONArray jsonArr = json.getJSONArray("term");

		// 时间范围聚合(必须条件)
		AggregationBuilder DateRange = AggregationBuilders.dateRange("agg")
				.field(jsonArr.getJSONObject(0).getString("term")).format("yyyy/MM/dd HH:mm:ss")
				.addRange(jsonArr.getJSONObject(0).getString("from"), jsonArr.getJSONObject(0).getString("to"));

		// 日期直方图聚合

		AggregationBuilder DateHistogram = null;
		if (json.containsKey("cut")) {
			DateHistogramInterval datehistogram = cutDate(json.getJSONObject("cut").getString("type"),
					json.getJSONObject("cut").getString("num"));
			if (datehistogram != null) {
				DateHistogram = AggregationBuilders.dateHistogram("agg")
						.field(jsonArr.getJSONObject(0).getString("term")).dateHistogramInterval(datehistogram);
			}
		}

		// 度量聚合
		AggregationBuilder aggregation = aggregation(json.getJSONObject("converge").getString("type"),
				json.getJSONObject("converge").getString("cterm"));

		if (DateHistogram != null && aggregation != null) {
			DateRange.subAggregation(DateHistogram.subAggregation(aggregation));
		}
		if (DateHistogram == null && aggregation != null) {
			DateRange.subAggregation(aggregation);
		}

		SearchRequestBuilder srb = client.prepareSearch(indexName).setTypes(indexName);
		// srb.setQuery(QueryBuilders.boolQuery().must(rangequerybuilder));
		srb.setQuery(builder(jsonArr));
		srb.addAggregation(DateRange);

		SearchResponse response = srb.execute().actionGet();

		Range agg = response.getAggregations().get("agg");

		// Histogram agg = response.getAggregations().get("agg");

		if (DateHistogram != null) {
			for (Range.Bucket entry : agg.getBuckets()) {

				Histogram agg2 = (Histogram) entry.getAggregations().asMap().get("agg");
				for (Histogram.Bucket entry2 : agg2.getBuckets()) {
					JSONObject Json = new JSONObject();
					DateTime key = (DateTime) entry2.getKey(); // Key
					// String keyAsString = entry2.getKeyAsString(); // Key as
					// String
					long docCount = entry2.getDocCount(); // Doc count
					double value = (double) entry2.getAggregations().asMap().get("agg").getProperty("value");

					if (docCount > 0) {
						Json.put("date", key.getYear() + "/" + key.getMonthOfYear() + "/" + key.getDayOfMonth() + " "
								+ key.getHourOfDay() + ":" + key.getMinuteOfHour() + ":" + key.getSecondOfMinute());
						Json.put("count", docCount);
						Json.put(json.getJSONObject("converge").getString("type") + "_value", value);
						jsonlist.add(Json);
						// System.out.println("date
						// [{"+key.getYear()+"/"+key.getMonthOfYear()+"/"+key.getDayOfMonth()+"
						// "+key.getHourOfDay()+":"+key.getMinuteOfHour()+":"+key.getSecondOfMinute()+"}],
						// doc_count [{"+docCount+"},
						// agg_value[{"+value+"}]]]");
					}
				}
			}
		} else {
			for (Range.Bucket entry : agg.getBuckets()) {
				JSONObject Json = new JSONObject();
				String key = (String) entry.getKey(); // Key
				// String keyAsString = entry.getKeyAsString(); // Key as String
				long docCount = entry.getDocCount(); // Doc count
				double value = (double) entry.getAggregations().asMap().get("agg").getProperty("value");

				if (docCount > 0) {
					Json.put("date", key);
					Json.put("count", docCount);
					Json.put(json.getJSONObject("converge").getString("type") + "_value", value);
					jsonlist.add(Json);
					// System.out.println("date [{"+key+"}], doc_count
					// [{"+docCount+"}, agg_value[{"+value+"}]]]");
				}
			}
		}
		return jsonlist;
	}

	/**
	 * 获取项目中数据总条数
	 * 
	 * @param client
	 * @param indexList
	 * @return
	 */
	public JSONObject queryGraphNotTime(final List indexList) {
		IndexTableEntity index = indexList.get(0);

		JSONObject Json = new JSONObject();

		SearchRequestBuilder srb = client.prepareSearch(index.getIndex()).setTypes(index.getIndex());

		SearchResponse response = srb.execute().actionGet();
		SearchHits searchHits = response.getHits();
		Json.put("count", searchHits.getTotalHits());
		return Json;
	}

	/**
	 * 分组聚合
	 * 
	 * @param client
	 * @param json
	 * @param indexList
	 * @return
	 */
	public JSONArray queryGraphTerms(final JSONObject json, final List indexList) {
		IndexTableEntity index = indexList.get(0);

		JSONArray jsonlist = new JSONArray();
		JSONArray jsonArr = json.getJSONArray("term");

		// 度量聚合
		AggregationBuilder aggregation = aggregation(json.getJSONObject("converge").getString("type"),
				json.getJSONObject("converge").getString("cterm"));

		SearchRequestBuilder srb = client.prepareSearch(index.getIndex()).setTypes(index.getIndex());
		// srb.setQuery(QueryBuilders.boolQuery().must(rangequerybuilder));
		srb.setQuery(builder(jsonArr));
		srb.addAggregation(aggregation);

		SearchResponse response = srb.execute().actionGet();

		Terms agg = response.getAggregations().get("agg");

		// Histogram agg = response.getAggregations().get("agg");

		for (Terms.Bucket entry : agg.getBuckets()) {
			JSONObject Json = new JSONObject();
			String key = (String) entry.getKey(); // Key
			String keyAsString = entry.getKeyAsString(); // Key as String
			long docCount = entry.getDocCount(); // Doc count

			if (docCount > 0) {
				Json.put("date", key);
				Json.put("count", docCount);
				Json.put(json.getJSONObject("converge").getString("cterm"), keyAsString);
				jsonlist.add(Json);
				// System.out.println("date [{"+key+"}], doc_count
				// [{"+docCount+"}, agg_value[{"+value+"}]]]");
			}
		}
		return jsonlist;
	}

	/**
	 * 分组聚合（无过滤条件）
	 * 
	 * @param client
	 * @param json
	 * @param indexList
	 * @return
	 */
	public JSONArray queryGraphTermsV1(final JSONObject json, final List indexList) {
		IndexTableEntity index = indexList.get(0);

		JSONArray jsonlist = new JSONArray();

		// 度量聚合
		AggregationBuilder aggregation = aggregation(json.getJSONObject("converge").getString("type"),
				json.getJSONObject("converge").getString("cterm"));

		SearchRequestBuilder srb = client.prepareSearch(index.getIndex()).setTypes(index.getIndex());
		srb.addAggregation(aggregation);

		SearchResponse response = srb.execute().actionGet();

		Terms agg = response.getAggregations().get("agg");

		// Histogram agg = response.getAggregations().get("agg");

		for (Terms.Bucket entry : agg.getBuckets()) {
			JSONObject Json = new JSONObject();
			String key = (String) entry.getKey(); // Key
			String keyAsString = entry.getKeyAsString(); // Key as String
			long docCount = entry.getDocCount(); // Doc count

			if (docCount > 0) {
				Json.put("date", key);
				Json.put("count", docCount);
				Json.put(json.getJSONObject("converge").getString("cterm"), keyAsString);
				jsonlist.add(Json);
				// System.out.println("date [{"+key+"}], doc_count
				// [{"+docCount+"}, agg_value[{"+value+"}]]]");
			}
		}
		return jsonlist;
	}

	// 分割采样
	public static DateHistogramInterval cutDate(final String dateType, final String dateNum) {
		if (StringUtils.isEmpty(dateType)) {
			LOGGER.info("缺少参数：dateType !");
		}
		if (StringUtils.isEmpty(dateNum)) {
			LOGGER.info("缺少参数：dateNum !");
		}

		DateHistogramInterval datehistogram = null;
		switch (dateType) {
		// 年
		case "YEAR":
			datehistogram = new DateHistogramInterval(dateNum + "y");
			break;
		// 季度
		case "QUARTER":
			datehistogram = new DateHistogramInterval(dateNum + "q");
			break;
		// 月
		case "MONTH":
			datehistogram = new DateHistogramInterval(dateNum + "M");
			break;
		// 周
		case "WEEK":
			datehistogram = new DateHistogramInterval(dateNum + "w");
			break;
		// 日
		case "DAY":
			datehistogram = new DateHistogramInterval(dateNum + "d");
			break;
		// 时
		case "HOUR":
			datehistogram = new DateHistogramInterval(dateNum + "h");
			break;
		// 分
		case "MINUTE":
			datehistogram = new DateHistogramInterval(dateNum + "m");
			break;
		// 秒
		case "SECOND":
			datehistogram = new DateHistogramInterval(dateNum + "s");
			break;
		}
		return datehistogram;
	}

	// 聚合类型
	public static AggregationBuilder aggregation(final String aggType, final String aggTerm) {
		if (StringUtils.isEmpty(aggType)) {
			LOGGER.info("缺少参数：aggType !");
		}
		if (StringUtils.isEmpty(aggTerm)) {
			LOGGER.info("缺少参数：aggTerm !");
		}

		AggregationBuilder agg = null;
		switch (aggType) {
		// 最小值
		case "MIN":
			agg = AggregationBuilders.min("agg").field(aggTerm);
			break;
		// 最大值
		case "MAX":
			agg = AggregationBuilders.max("agg").field(aggTerm);
			break;
		// 求和
		case "SUM":
			agg = AggregationBuilders.sum("agg").field(aggTerm);
			break;
		// 平均值
		case "AVG":
			agg = AggregationBuilders.avg("agg").field(aggTerm);
			break;
		// 计数
		case "COUNT":
			agg = AggregationBuilders.count("agg").field(aggTerm);
			break;
		// 统计数据
		case "TERMS":
			agg = AggregationBuilders.terms("agg").field(aggTerm).size(10000);
			break;
		}
		return agg;
	}

	// 过滤条件
	public static BoolQueryBuilder builder(final JSONArray jsonArr) {
		BoolQueryBuilder boolQueryuilder = QueryBuilders.boolQuery();
		for (int i = 1; i < jsonArr.size(); i++) {
			switch (jsonArr.getJSONObject(i).getString("sign")) {
			// 等于
			case "=":
				boolQueryuilder.must(QueryBuilders.matchPhraseQuery(jsonArr.getJSONObject(i).getString("term"),
						jsonArr.getJSONObject(i).getString("data")));
				break;
			// 大于
			case ">":
				boolQueryuilder.must(QueryBuilders.rangeQuery(jsonArr.getJSONObject(i).getString("term"))
						.gt(jsonArr.getJSONObject(i).getString("data")));
				break;
			// 大于等于
			case ">=":
				boolQueryuilder.must(QueryBuilders.rangeQuery(jsonArr.getJSONObject(i).getString("term"))
						.gte(jsonArr.getJSONObject(i).getString("data")));
				break;
			// 小于
			case "<":
				boolQueryuilder.must(QueryBuilders.rangeQuery(jsonArr.getJSONObject(i).getString("term"))
						.lt(jsonArr.getJSONObject(i).getString("data")));
				break;
			// 小于等于
			case "<=":
				boolQueryuilder.must(QueryBuilders.rangeQuery(jsonArr.getJSONObject(i).getString("term"))
						.lte(jsonArr.getJSONObject(i).getString("data")));
				break;
			case "!=":
				boolQueryuilder.mustNot(QueryBuilders.matchPhraseQuery(jsonArr.getJSONObject(i).getString("term"),
						jsonArr.getJSONObject(i).getString("data")));
				break;
			}
		}
		return boolQueryuilder;
	}

	public String queryData(final String index, final String lastDate) {
		final SearchRequestBuilder requestBuilder = client.prepareSearch(index).setTypes(index);

		// 组装条件
		QueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("runtime").gte(lastDate)
				.lte(DateUtils.formateDate(new Date()));
		BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
		boolQueryBuilder.filter(rangeQueryBuilder);
		requestBuilder.setQuery(boolQueryBuilder);
		requestBuilder.setSize(10000);

		// 4.执行dsl
		final SearchResponse response = requestBuilder.execute().actionGet();

		// 解析
		List list = new ArrayList<>();
		SearchHits searchHits = response.getHits();
		SearchHit[] hits = searchHits.getHits();
		for (int i = 0; i < hits.length; i++) {
			String data = hits[i].getSourceAsString();
			list.add(data);
		}
		return list.toString();
	}

	public String queryData2(final String index, final Integer size) {
		final SearchRequestBuilder requestBuilder = client.prepareSearch(index).setTypes(index);
		// 组装条件
		QueryBuilder matchAllQuery = QueryBuilders.matchAllQuery();
		requestBuilder.setQuery(matchAllQuery);
		if (size == null) {
			requestBuilder.setSize(10000);
		} else {
			requestBuilder.setSize(size);
		}

		// 4.执行dsl
		final SearchResponse response = requestBuilder.execute().actionGet();

		// 解析
		List list = new ArrayList<>();
		SearchHits searchHits = response.getHits();
		SearchHit[] hits = searchHits.getHits();
		for (int i = 0; i < hits.length; i++) {
			String data = hits[i].getSourceAsString();
			list.add(data);
		}
		return list.toString();
	}

	/**
	 * @param index
	 *            index
	 * @param lastDate
	 *            当前时间
	 * @return
	 */
	public String queryProjectLastTime(final String index, final String lastDate) {
		final SearchRequestBuilder requestBuilder = client.prepareSearch(index).setTypes(index);

		// 组装条件
		requestBuilder.setSize(1);// 查询一条
		requestBuilder.addSort(TIMESTAMP, SortOrder.DESC);// 倒叙
		// 查询条件
		final BoolQueryBuilder boolQueryuilder = QueryBuilders.boolQuery();
		final QueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(TIMESTAMP).to(lastDate);
		boolQueryuilder.filter(rangeQueryBuilder);
		requestBuilder.setQuery(boolQueryuilder);

		LOGGER.info("SearchRequestBuilder查询语句：" + requestBuilder.toString());

		// 4.执行dsl
		final SearchResponse response = requestBuilder.execute().actionGet();

		String lastTime = null;
		final SearchHits hits = response.getHits();
		if (hits != null && hits.totalHits() > 0) {
			final SearchHit[] searchHits = hits.hits();
			final Map map = searchHits[0].getSource();
			if (map.containsKey(TIMESTAMP)) {
				lastTime = map.get(TIMESTAMP).toString();
			}

		}
		return lastTime;
	}

	/**
	 * 根据条件删除索引
	 * 
	 * @param indexName
	 *            索引
	 * @param startTime
	 *            开始时间
	 * @param endTime
	 *            结束时间
	 * @param deviceId
	 *            设备Id
	 * @param metric
	 *            测点
	 */
	public void dropPhyTableByCondition(final String indexName, final String startTime, final String endTime,
			final String deviceId, final String metric) {
		try {

			// 查询条件
			final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
			final QueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(TIMESTAMP).from(startTime).to(endTime);
			boolQueryBuilder.filter(rangeQueryBuilder);
			final QueryBuilder tagTermQuery = QueryBuilders.termQuery("tag", 1);
			boolQueryBuilder.filter(tagTermQuery);
			final QueryBuilder deviceIdTermQuery = QueryBuilders.termQuery("deviceid", deviceId);
			boolQueryBuilder.filter(deviceIdTermQuery);
			final QueryBuilder metricTermQuery = QueryBuilders.termQuery("metric", metric);
			boolQueryBuilder.filter(metricTermQuery);

			BulkIndexByScrollResponse dResponse = new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
					.filter(boolQueryBuilder).source(indexName).get();
			long deleted = dResponse.getDeleted();

			if (deleted >= 0) {
				LOGGER.info("delete index " + indexName + "  successfully!");
			} else {
				LOGGER.error("dropPhyTable(): Fail to delete index " + indexName);
				// throw new RuntimeException(msg);
			}
		} catch (Exception e) {
			LOGGER.error("dropPhyTable(): Fail to delete index " + indexName);
			throw new RuntimeException("删除索引数据失败: " + indexName, e);
		}

	}

	/**
	 * 数据更新
	 * 
	 * @param resultList
	 *            voList
	 * @return ResultMsg 结果
	 */
	public ResultMsg updateIndex(final String index, final JSONArray resultList) {
		final ResultMsg resultMsg = new ResultMsg();
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		int count = 0;
		int len = resultList.size();
		try {
			for (int i = 0; i < len; i++) {
				final XContentBuilder builder = XContentFactory.jsonBuilder().startObject();

				JSONObject json = resultList.getJSONObject(i);
				final String id = json.getString("id");
				json.remove("id");

				for (String key : json.keySet()) {
					builder.field(key, json.get(key));
				}

				builder.endObject();

				UpdateRequestBuilder requestBuilder = client.prepareUpdate(index, index, id).setDoc(builder);
				// IndexRequestBuilder requestBuilder = client.prepareIndex(
				// "sampling", indexName, id).setSource(builder);
				bulkRequest.add(requestBuilder);
				count++;

				if (count % BULK_SIZE == 0) {
					BulkResponse bulkResponse = bulkRequest.execute().actionGet();
					if (bulkResponse.hasFailures()) {
						LOGGER.error("批量更新索引数据失败: " + indexName);
						LOGGER.error("批量更新索引数据失败: " + bulkResponse.buildFailureMessage());

						resultMsg.setHasFailures(true);

						BulkItemResponse[] responses = bulkResponse.getItems();
						for (int k = 0; k < responses.length; k++) {
							BulkItemResponse response = responses[k];
							if (response.isFailed()) {
								ErrorDetail errorDetail = new ErrorDetail();
								errorDetail.setIndex(k + (i / BULK_SIZE));
								errorDetail.setId(response.getId());
								errorDetail.setMsg(response.getFailureMessage());

								resultMsg.addError(errorDetail);
							}
						}
					}

					bulkRequest = client.prepareBulk();
					bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
					count = 0;
				}
			}

			if (count > 0) {
				BulkResponse bulkResponse = bulkRequest.execute().actionGet();
				if (bulkResponse.hasFailures()) {
					LOGGER.error("批量更新索引数据失败: " + indexName);
					LOGGER.error("批量更新索引数据失败: " + bulkResponse.buildFailureMessage());

					resultMsg.setHasFailures(true);

					BulkItemResponse[] responses = bulkResponse.getItems();
					for (int k = 0; k < responses.length; k++) {
						BulkItemResponse response = responses[k];
						if (response.isFailed()) {
							ErrorDetail errorDetail = new ErrorDetail();
							errorDetail.setIndex(k + (len / BULK_SIZE));
							errorDetail.setId(response.getId());
							errorDetail.setMsg(response.getFailureMessage());

							resultMsg.addError(errorDetail);
						}
					}
				}
			}

			return resultMsg;
		} catch (Exception e) {
			LOGGER.error("批量更新索引数据失败: " + indexName);
			throw new RuntimeException("批量更新索引数据失败: " + indexName, e);
		} finally {
			bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
		}

	}

	/**
	 * 数据写入
	 * 
	 * @param resultList
	 * @return ResultMsg 结果
	 */
	public ResultMsg importIndex(final String index, final JSONArray resultList) {
		final ResultMsg resultMsg = new ResultMsg();
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);

		int count = 0;
		int len = resultList.size();
		try {
			for (int i = 0; i < len; i++) {
				final XContentBuilder builder = XContentFactory.jsonBuilder().startObject();

				JSONObject json = resultList.getJSONObject(i);
				for (String key : json.keySet()) {
					builder.field(key, json.get(key));
				}
				builder.endObject();

				IndexRequestBuilder requestBuilder = client.prepareIndex(index, index).setSource(builder);
				bulkRequest.add(requestBuilder);
				count++;

				if (count % BULK_SIZE == 0) {
					BulkResponse bulkResponse = bulkRequest.execute().actionGet();
					if (bulkResponse.hasFailures()) {
						LOGGER.error("批量写入索引数据失败: " + indexName);
						LOGGER.error("批量写入索引数据失败: " + bulkResponse.buildFailureMessage());

						resultMsg.setHasFailures(true);

						BulkItemResponse[] responses = bulkResponse.getItems();
						for (int k = 0; k < responses.length; k++) {
							BulkItemResponse response = responses[k];
							if (response.isFailed()) {
								ErrorDetail errorDetail = new ErrorDetail();
								errorDetail.setIndex(k + (i / BULK_SIZE));
								errorDetail.setId(response.getId());
								errorDetail.setMsg(response.getFailureMessage());

								resultMsg.addError(errorDetail);
							}
						}
					}

					bulkRequest = client.prepareBulk();
					bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
					count = 0;
				}
			}

			if (count > 0) {
				BulkResponse bulkResponse = bulkRequest.execute().actionGet();
				if (bulkResponse.hasFailures()) {
					LOGGER.error("批量写入索引数据失败: " + indexName);
					LOGGER.error("批量写入索引数据失败: " + bulkResponse.buildFailureMessage());

					resultMsg.setHasFailures(true);

					BulkItemResponse[] responses = bulkResponse.getItems();
					for (int k = 0; k < responses.length; k++) {
						BulkItemResponse response = responses[k];
						if (response.isFailed()) {
							ErrorDetail errorDetail = new ErrorDetail();
							errorDetail.setIndex(k + (len / BULK_SIZE));
							errorDetail.setId(response.getId());
							errorDetail.setMsg(response.getFailureMessage());

							resultMsg.addError(errorDetail);
						}
					}
				}
			}

			return resultMsg;
		} catch (Exception e) {
			LOGGER.error("批量写入索引数据失败: " + indexName);
			throw new RuntimeException("批量写入索引数据失败: " + indexName, e);
		} finally {
			bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
		}

	}

	public JSONArray queryToCompletion(final String index, final String startTime, final String endTime,
			final String deviceId, final String metric) {
		/*
		 * 查询出需要更新的数据
		 */
		JSONArray resultList = new JSONArray();
		final SearchRequestBuilder requestBuilder1 = client.prepareSearch(index).setTypes(index);
		// 查询条件
		final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
		final QueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(TIMESTAMP).from(startTime).to(endTime);
		boolQueryBuilder.filter(rangeQueryBuilder);
		final QueryBuilder deviceIdTermQuery = QueryBuilders.termQuery("deviceid", deviceId);
		boolQueryBuilder.filter(deviceIdTermQuery);
		final QueryBuilder metricTermQuery = QueryBuilders.termQuery("metric", metric);
		boolQueryBuilder.filter(metricTermQuery);
		requestBuilder1.setQuery(boolQueryBuilder);

		LOGGER.info("SearchRequestBuilder查询语句：" + requestBuilder1.toString());

		// 4.执行dsl
		final SearchResponse response1 = requestBuilder1.execute().actionGet();

		final SearchHits hits = response1.getHits();
		if (hits != null && hits.totalHits() > 0) {
			final SearchHit[] searchHits = hits.hits();
			for (int i = 0; i < searchHits.length; i++) {
				JSONObject json = new JSONObject();
				json.put("id", searchHits[i].getId());
				json.put("tag", 0);
				resultList.add(json);
			}
		}
		return resultList;
	}
}