package com.ncs.dao
import java.net.InetAddress
import java.util.ArrayList
import java.util.List
import java.util.Map
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest
import org.elasticsearch.action.bulk.BulkRequestBuilder
import org.elasticsearch.action.delete.DeleteResponse
import org.elasticsearch.action.get.GetResponse
import org.elasticsearch.action.index.IndexRequestBuilder
import org.elasticsearch.action.index.IndexResponse
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.update.UpdateResponse
import org.elasticsearch.client.Requests
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.InetSocketTransportAddress
import org.elasticsearch.common.xcontent.XContentBuilder
import org.elasticsearch.common.xcontent.XContentFactory
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.FuzzyQueryBuilder
import org.elasticsearch.index.query.MatchAllQueryBuilder
import org.elasticsearch.index.query.PrefixQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.query.QueryStringQueryBuilder
import org.elasticsearch.index.query.RangeQueryBuilder
import org.elasticsearch.index.query.SpanFirstQueryBuilder
import org.elasticsearch.index.query.WildcardQueryBuilder
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.SearchHits
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import com.alibaba.fastjson.JSON
import com.google.common.collect.Maps
import com.ncs.Conf
import com.ncs.entity.ESEntity
public class ElasticSearchDAO {
private static final Logger logger = LoggerFactory
.getLogger(ElasticSearchDAO.class)
// 声明静态配置
private static TransportClient client = null
private volatile static BulkRequestBuilder prepareBulk
// 初始化静态配置
static {
// es 2.4.4的连接方式
String EsHosts = Conf.get("es.hosts")
Settings settings = Settings.settingsBuilder()
.put("cluster.name", Conf.get("es.cluster.name"))// 设置集群名称
.put("tclient.transport.sniff", true).build()
// 获取客户端
client = TransportClient.builder().settings(settings).build()
String[] nodes = EsHosts.split(",")
for (String node : nodes) {
if (node.length() > 0) {// 跳过为空的node(当开头、结尾有逗号或多个连续逗号时会出现空node)
String[] hostPort = node.split(":")
try {
client.addTransportAddress(new InetSocketTransportAddress(
InetAddress.getByName(hostPort[0]), Integer.parseInt(hostPort[1])))
} catch (Exception e) {
e.printStackTrace()
}
}
}
}
public static void closeClient() {
client.close()
logger.info("client closed!")
}
public static void createIndexName(String indexname) {
client.admin().indices().prepareCreate(indexname).execute().actionGet()
logger.info("create index " + indexname + " success!")
}
public static void createMapping(String indexname, String mappingType)
throws Exception {
new XContentFactory()
XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
// .startObject("_ttl").field("enable", false).endObject()
.startObject("properties")
.startObject("domain").field("type", "string")
.field("store", "yes").field("index", "not_analyzed")
.endObject()
.startObject("rowKeys").field("type", "string")
.field("store", "yes").field("index", "not_analyzed")
.endObject()
// .startObject("clazz").field("type", "string")
// .field("store", "yes").field("index", "not_analyzed")
// .endObject()
// .startObject("type").field("type", "string")
// .field("store", "yes").field("index", "not_analyzed")
// .endObject()
// .startObject("rdata").field("type", "string")
// .field("store", "yes").field("index", "not_analyzed")
// .endObject()
// .startObject("ispId")
// .field("type", "integer").field("store", "yes")
// .field("index", "not_analyzed")
// .endObject()
// .startObject("version")
// .field("type", "long").field("store", "yes")
// .field("index", "not_analyzed")
// .endObject()
.endObject().endObject()
PutMappingRequest mapping = Requests.putMappingRequest(indexname)
.type(mappingType).source(builder)
client.admin().indices().putMapping(mapping).actionGet()
logger.info("create mapping " + mappingType + " success!")
}
public static void createIndexResponse(String indexname, String type,
List jsondata) {
// 创建索引库 需要注意的是.setRefresh(true)这里一定要设置,否则第一次建立索引查找不到数据
IndexRequestBuilder requestBuilder = client.prepareIndex(indexname,
type).setRefresh(true)
for (int i = 0
IndexResponse actionGet = requestBuilder.setId(i + 1 + "").setSource(jsondata.get(i))
.execute().actionGet()
logger.info("response.getVersion():" + actionGet.getVersion())
}
}
public static IndexRequestBuilder createIndexResponse(String indexname,
String type, String jsondata, String id) {
IndexRequestBuilder response = client.prepareIndex(indexname, type)
// 必须为对象单独指定ID
.setId(id).setSource(jsondata)
// 多次index这个版本号会变
// logger.info("response.getVersion():" + response.getVersion())
return response
}
public static void creatIndexResponseBulk(String indexname,
String type, String jsondata, String id) throws Exception{
prepareBulk = client.prepareBulk()
prepareBulk = prepareBulk.add(ElasticSearchDAO.createIndexResponse(indexname, type, jsondata, id))
}
public static void execute() throws Exception {
// 每1000条提交一次
prepareBulk.execute().actionGet()
}
public static DeleteResponse deleteResponse(String indexname, String type,
String id) {
DeleteResponse response = client.prepareDelete(indexname, type, id)
.execute().actionGet()
logger.info(response.getId() + " delete success!")
return response
}
public static UpdateResponse updataResponse(String indexname, String type,
String id, String cloumn, String value) {
Map params = Maps.newHashMap()
params.put(cloumn, value)
UpdateResponse response = client.prepareUpdate(indexname, type, id)
// .setScript("ctx._source." + cloumn + "=" + cloumn,
// ScriptType.INLINE).setScriptParams(params)
.execute().actionGet()
logger.info("updata success!")
return response
}
public static GetResponse getResponse(String indexname, String type,
String id) {
GetResponse response = client.prepareGet(indexname, type, id).execute()
.actionGet()
logger.info("response.getId():" + response.getId() + "response.getSourceAsString():" + response.getSourceAsString())
return response
}
public static List searcherESEntitys(
QueryBuilder queryBuilder, String indexname, String type) {
List list = new ArrayList()
// 执行搜索
SearchResponse searchResponse = client.prepareSearch(indexname)
.setTypes(type).setQuery(queryBuilder).execute().actionGet()
// 获取搜索结果
SearchHits hits = searchResponse.getHits()
System.out.println("查询到记录数=" + hits.getTotalHits())
SearchHit[] searchHists = hits.getHits()
String domain
String rowKeys
// 遍历将搜索结果封装成Record对象,添加到list集合中
if (searchHists.length > 0) {
for (SearchHit hit : searchHists) {
domain = (String) hit.getSource().get("domain")
rowKeys = (String) hit.getSource().get("rowKeys")
list.add(new ESEntity(domain, rowKeys))
}
}
return list
}
public static void intoEs(List esEntities,
String indexname, String type) {
String receiveString
try {
// 将集合对象遍历存入es中
for (ESEntity esEntity : esEntities) {
// 将对象转换为json字符串
receiveString = JSON.toJSONString(esEntity)
// 批量插入es之前的准备
ElasticSearchDAO.creatIndexResponseBulk(indexname, type, receiveString, esEntity.getDomain())
}
// 批量插入es中
ElasticSearchDAO.execute()
logger.info(esEntities.size() + " row eSEntity has inesrt into es!")
} catch (Exception e) {
logger.error(" inesrt into es error!" + e.toString())
e.printStackTrace()
}
}
@SuppressWarnings({ "unused" })
public static void main(String[] args) {
String indexname = "zone"
String type = "records"
// 获得需要插入的数据
// List jsondata = Zone.getInitJsonData()
// 创建index并插入数据,(只是插入数据时需要)
// ElasticSearchJavaAPI.createIndexResponse(indexname, type, jsondata)
// 查询条件
// //////////////////////////////////////////////////////////////////////////////////////
// 按照其中某一项进行查询
QueryBuilder queryBuilder = QueryBuilders.termQuery("owner", "sex2")
// content为field,test为查询内容.
// 其中must表示必须满足,mustNot表示必须不满足,should表示可有可无
// 如果bool查询语句中不存在must,则必须至少有一个should查询,同时可以通过minimum_should_match参数来设置至少需要满足的should个数.
QueryBuilder qb = QueryBuilders
.boolQuery()
.must(QueryBuilders.termQuery("rdata", "ns1.101domain.com"))
// .must(QueryBuilders.termQuery("rdata", "ns1.sdc.org.cn"))
.mustNot(
QueryBuilders
.termQuery("rdata", "nsgbr.comlaude.co.uk"))
.should(QueryBuilders.termQuery("rdata", "ns1.sdc.org.cn"))
// 查询汉字时需要将汉字拆分开,因为elasticsearch 里默认的IK分词器是会将每一个中文都进行了分词的切割
QueryBuilder queryBuilder3 = QueryBuilders.termQuery("owner", "洞")
QueryBuilder queryBuilder4 = QueryBuilders.termQuery("owner", "头")
BoolQueryBuilder bool = QueryBuilders.boolQuery()
bool.must(queryBuilder3)
bool.must(queryBuilder4)
// 根据前缀查询
PrefixQueryBuilder prefixQuery = QueryBuilders.prefixQuery("domain", "163")
// 通配符查询
WildcardQueryBuilder wildcardQuery = QueryBuilders.wildcardQuery(
"rdata", "?ns")
// 根据提供的字符串作为前缀进行查询
FuzzyQueryBuilder fuzzyQuery = QueryBuilders.fuzzyQuery("name", "sex")
// 范围查询
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("rdata")
.from(0).to(10)
// 查询所有
MatchAllQueryBuilder matchAllQuery = QueryBuilders.matchAllQuery()
// 根据id查询
QueryBuilder queryBuilder2 = QueryBuilders.boolQuery().must(
QueryBuilders.termQuery("id", 1))
// 只匹配在Field开头出现的, 数字表示从开头起的几个单词内查询, 则此查询意思是:ns中的开头3个单词内,nsgbr是否能查询到
SpanFirstQueryBuilder spanFirstQuery = QueryBuilders.spanFirstQuery(
QueryBuilders.spanTermQuery("rdata", "nsgbr"), // Query
3 // Max End position
)
// String查询
QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery("sex")
// 文本查询
// MatchQueryBuilder textPhrase = QueryBuilders.textPhrase("", null)
// /////////////////////////////////////////////////////////////////////////////
// List result = ElasticSearchDAO.searcherDNResourceRecords(prefixQuery, indexname,
// type)
// for (int i = 0
// DNResourceRecord record = result.get(i)
// System.out.println("(" + record.getId() + ") 主机名:" + record.getOwner()+ "\t域名:" + record.getRdata())
// }
//
// GetResponse response = ElasticSearchDAO.getResponse(indexname, type, "2")
// List registerInfos = ElasticSearchDAO.searcherDNRegisterInfos(prefixQuery, indexname, type)
// for (DNRegisterInfo dnRegisterInfo : registerInfos) {
// System.out.println("主机名:" + dnRegisterInfo.getName() + " 注册商姓名:"
// + dnRegisterInfo.getRegistrantName() + " " + dnRegisterInfo.getUpdatedDate())
// }
// List list = ElasticSearchDAO.searcherESEntitys(prefixQuery, indexname, type)
// for (ESEntity esEntity : list) {
// System.out.println(esEntity)
// }
// 创建索引和mapping
ElasticSearchDAO.createIndexName("zone")
try {
ElasticSearchDAO.createMapping("zone", "records")
} catch (Exception e) {
e.printStackTrace()
}
// System.out.println("********************************")
// ElasticSearchDAO.deleteResponse(indexname, type, "sex4.tld.")
// ElasticSearchDAO.updataResponse(indexname, type, "sex4.tld.", "id", "sex4.tld._info")
ElasticSearchDAO.closeClient()
}
}