Elasticsearch+python学习
爬虫中建立moudle文件夹用于存放elasticsearch基本数据操作命令（建表）
from datetime import datetime
from elasticsearch_dsl import DocType, Date, Nested, Boolean,analyzer, InnerObjectWrapper, Completion, Keyword, Text, Integer
from elasticsearch_dsl.connections import connections
#创建服务器链接,非常终于
connections.create_connection(hosts=["localhost"])
#定义数据类，继承DocType,定义各个字段数据类型，在from elasticsearch_dsl import中导入需要的数据类型，包括字符串，整型，布尔等等
class LagouType(DocType):
job_name = Text(analyzer="ik_max_word")
company = Text(analyzer="ik_max_word")
url = Keyword()
job_id = Keyword()
salary = Text(analyzer="ik_max_word")
city = Keyword()
experience = Text(analyzer="ik_max_word")
education = Text(analyzer="ik_max_word")
job_type = Keyword()
label = Text(analyzer="ik_max_word")
job_benefit = Text(analyzer="ik_max_word")
job_description = Text(analyzer="ik_max_word")
addr = Text(analyzer="ik_max_word")
publish_time = Text(analyzer="ik_max_word")
crawl_time = Date()
#建立链接的index和doc，在类中建立类，必须是Meta类，用于传入index值和type（表）值
class Meta:
index = "lagou"
doc_type = "job"
if __name__ == "__main__":
#调用init()方法建立映射（mappings）
LagouType.init()
在pipeline中定制与Elasticsearch连接
1.直接写在pipeline中，但是爬去的item不一定存入elasticsearch中或某数据库中，并且值内容不一，容易混乱，配置性低
#pipeline中写入
class Elasticsearch_pipeline(object):
def __init__(self):
pass
def process_item(self,item,spider):
lagou = LagouType()
lagou.job_name = item['job_name']
lagou.company = item['company']
lagou.url = item['url']
lagou.job_id = item['job_id']
lagou.salary = item['salary']
lagou.city = item['city']
lagou.experience = item['experience']
lagou.education = item['education']
lagou.job_type = item['job_type']
lagou.label = item['label']
lagou.job_benefit = item['job_benefit']
lagou.job_description = item['job_description']
lagou.addr = item['addr']
lagou.publish_time = item['publish_time']
lagou.crawl_time = item['crawl_time']
lagou.save()
return item
2.在item中定制save_to_elasticsearch接口，并在pipeline中调用item方法，增强item的可配置性
#item方法
def save_to_elasticsearch(self):
# 继承类
lagou = LagouType()
lagou.job_name = self['job_name']
lagou.company = self['company']
lagou.url = self['url']
lagou.job_id = self['job_id']
lagou.salary = self['salary']
lagou.city = self['city']
lagou.experience = self['experience']
lagou.education = self['education']
lagou.job_type = self['job_type']
lagou.label = self['label']
lagou.job_benefit = self['job_benefit']
lagou.job_description = self['job_description']
lagou.addr = self['addr']
lagou.publish_time = self['publish_time']
lagou.crawl_time = self['crawl_time']
lagou.save()
#pipeline调用
class Elasticsearch_pipeline(object):
def __init__(self):
pass
#在process_item中调用item的方法（item.save_to_elasticsearch()）
def process_item(self,item,spider):
item.save_to_elasticsearch()
return item
#settings中开启item_pipeline
ITEM_PIPELINES = {
'lagou_spider.pipelines.Elasticsearch_pipeline': 1
}
#settings中开启item_pipeline
ITEM_PIPELINES = {
'lagou_spider.pipelines.Elasticsearch_pipeline': 1
}
记录Elasticsearch与Python的各种查询操作（基本与kibana中的elasticsearch操作相同可以照搬）
重点“ from elasticsearch import Elasticsearch”
Elasticsearch的查询API接口：
client = Elasticsearch()
response = client.search(select sentence……)
class Elasticsearch_Option:
def __init__(self):
pass
#注意点1：注意大小写，进行分词分析时，elasticsearch的分词器会把自动把所有词变成小写
#match 用法，对 match 传入的值进行分词，符合分词结果的都可以检索到
def match_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"match": {
'title':'C++后端工程师'
}
}
}
)
#term 用法，不对 term 传入的值进行分词
def term_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"term": {
'salary_min':'2000000'
}
}
}
)
#terms 用法，可传入列表，符合列表内的值都可以检索到
def terms_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"terms": {
'title': ['python','java','c++'] #千万注意大小写
}
}
}
)
#from 和 size 的用法
def from_size_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"match": {
'title':'工程师'
}
},
"from":0,
"size":4
}
)
#match_all操作
def match_all_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"match_all": {
}
}
}
)
# match_phrase 短语查询
def match_phrase_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"match_phrase": {
"title": 'python研发工程师'
}
}
}
)
for i in response['hits']['hits']:
print(i['_source'])
#multi_match查询，单一查询条件查询多列（fields）
def multi_match_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"multi_match": {
"query":"深圳",
"fields": ['title','city'] #查询 fields 多个字段中，只要有：query查询内容的关键字的就查询出来。
}
}
}
)
#仔细留意response返回结构
for i in response['hits']['hits']:
print(i['_source'])
#排序操作
def sort_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"match_all":{}
},
"sort":{
"comment":{ #sort下面先制定需要排序的栏
"order": "asc"
}
}
}
)
# 仔细留意response返回结构
for i in response['hits']['hits']:
print(i['_source'])
#范围查询，gte：大于等于； gt：大于； lte：小于等于； lt：小于； boots：表示权重
def range_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"range":{
"comment":{ #range下面是要确定范围的field
"gt": 15,
"lt": 20
}
}
}
}
)
# 仔细留意response返回结构
for i in response['hits']['hits']:
print(i['_source'])
#wildcard,模糊查询
def wildcard_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"wildcard":{
"title":{ #range下面是要确定范围的field
"value":"pyth*n" # "*" 标识通配
}
}
}
}
)
# 仔细留意response返回结构
for i in response['hits']['hits']:
print(i['_source'])
# bool查询
# filter：字段过滤并且不参与打分，过滤掉非数组内的内容
# must：满足数组中所有的条件，“与”
# should：数组中的查询条件满足一个或多个，“或”
# must_not：数组中的查询条件一个都不能去满足，“非”
def bool_option(self):
client = Elasticsearch()
response = client.search(
index="lagou",
body={
"query": {
"bool": {
"must": [{
"match_all":{}
}],
"filter": {
"term": {
"title": '工程师'
}
},
"must_not": [{
"match": {
"comment": 16
}
}],
"should": [{
"match": {
"title": 'c'
}
}]
}
}
}
)
# 仔细留意response返回结构
for i in response['hits']['hits']:
print(i['_source'])