python连接elasticsearch
问题一：urllib3.exceptions.ProtocolError: (‘Connection aborted.’, RemoteDisconnected(‘Remote end closed connection without response’))
协议写错了，是https
问题一：SSLError([SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain
import ssl
from elasticsearch import Elasticsearch
# 不验证ssl_context=context，6.1.3版本有用
# 6.1.3版本设置verify_certs=False没用
# 因为` verify_certs=False`在后续创建连接时还是会使用默认的ssl加载，需设置context才有用
context = ssl._create_unverified_context()
client = Elasticsearch(hosts="https://1.1.1.1:9200/", http_auth=('用户名', '密码'), ssl_context=context)
# 不验证 verify_certs=False ，8版本有用
# 8版本设置ssl_context没用
# 因为8版本直接使用verify_certs这个变量进行控制，即使传参时传了不需要验证的上下文，他自己还会根据verify_certs参数，
# 如果它是true就会更新对应的数据，False时，直接创建一个不需要验证的上下文，连接的时候把这个上下文传递过去，
# 8版本的在一开始transport传参时就进行了判断创建
client = Elasticsearch(hosts="https://1.1.1.1:9200/", http_auth=('用户名', '密码'), verify_certs=False)
结论
1、elasticsearch Python 6.1.3 使用verify_certs=False时，不会创建ssl上下文，等到连接时发现没有上下文会自动创建一个默认的上下文，因此指定了上下文会起作用。
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\connection\http_urllib3.py
class Urllib3HttpConnection(Connection):
"""
Default connection class using the `urllib3` library and the http protocol.
:arg host: hostname of the node (default: localhost)
:arg port: port to use (integer, default: 9200)
:arg url_prefix: optional url prefix for elasticsearch
:arg timeout: default timeout in seconds (float, default: 10)
:arg http_auth: optional http auth information as either ':' separated
string or a tuple
:arg use_ssl: use ssl for the connection if `True`
:arg verify_certs: whether to verify SSL certificates
:arg ca_certs: optional path to CA bundle.
See https://urllib3.readthedocs.io/en/latest/security.html#using-certifi-with-urllib3
for instructions how to get default set
:arg client_cert: path to the file containing the private key and the
certificate, or cert only if using client_key
:arg client_key: path to the file containing the private key if using
separate cert and key files (client_cert will contain only the cert)
:arg ssl_version: version of the SSL protocol to use. Choices are:
SSLv23 (default) SSLv2 SSLv3 TLSv1 (see ``PROTOCOL_*`` constants in the
``ssl`` module for exact options for your environment).
:arg ssl_assert_hostname: use hostname verification if not `False`
:arg ssl_assert_fingerprint: verify the supplied certificate fingerprint if not `None`
:arg maxsize: the number of connections which will be kept open to this
host. See https://urllib3.readthedocs.io/en/1.4/pools.html#api for more
information.
:arg headers: any custom http headers to be add to requests
:arg http_compress: Use gzip compression
"""
def __init__(self, host='localhost', port=9200, http_auth=None,
use_ssl=False, verify_certs=VERIFY_CERTS_DEFAULT, ca_certs=None, client_cert=None,
client_key=None, ssl_version=None, ssl_assert_hostname=None,
ssl_assert_fingerprint=None, maxsize=10, headers=None, ssl_context=None, http_compress=False, **kwargs):
'''省略'''
kw = {}
# if providing an SSL context, raise error if any other SSL related flag is used
if ssl_context and ( (verify_certs is not VERIFY_CERTS_DEFAULT) or ca_certs
or client_cert or client_key or ssl_version):
warnings.warn("When using `ssl_context`, all other SSL related kwargs are ignored")
# if ssl_context provided use SSL by default
if ssl_context and self.use_ssl:
pool_class = urllib3.HTTPSConnectionPool
kw.update({
'assert_fingerprint': ssl_assert_fingerprint,
'ssl_context': ssl_context,
})
elif self.use_ssl:
pool_class = urllib3.HTTPSConnectionPool
kw.update({
'ssl_version': ssl_version,
'assert_hostname': ssl_assert_hostname,
'assert_fingerprint': ssl_assert_fingerprint,
})
# If `verify_certs` is sentinal value, default `verify_certs` to `True`
if verify_certs is VERIFY_CERTS_DEFAULT:
verify_certs = True
ca_certs = CA_CERTS if ca_certs is None else ca_certs
# verify_certs为false则不会更新任何ssl的信息，下边又因为没有cert_reqs和上下文变量，
# 导致创建了默认的上下文变量即VerifyMode.CERT_REQUIRED
if verify_certs:
if not ca_certs:
raise ImproperlyConfigured("Root certificates are missing for certificate "
"validation. Either pass them in using the ca_certs parameter or "
"install certifi to use it automatically.")
kw.update({
'cert_reqs': 'CERT_REQUIRED',
'ca_certs': ca_certs,
'cert_file': client_cert,
'key_file': client_key,
})
else:
warnings.warn(
'Connecting to %s using SSL with verify_certs=False is insecure.' % host)
# 该处会调用下边的HTTPSConnectionPool
self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize, **kw)
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py
class HTTPSConnectionPool(HTTPConnectionPool):
"""
Same as :class:`.HTTPConnectionPool`, but HTTPS.
:class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
``assert_hostname`` and ``host`` in this order to verify connections.
If ``assert_hostname`` is False, no verification is done.
The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
the connection socket into an SSL socket.
"""
scheme = "https"
ConnectionCls = HTTPSConnection
def __init__(
self,
host,
port=None,
strict=False,
timeout=Timeout.DEFAULT_TIMEOUT,
maxsize=1,
block=False,
headers=None,
retries=None,
_proxy=None,
_proxy_headers=None,
key_file=None,
cert_file=None,
cert_reqs=None,
key_password=None,
ca_certs=None,
ssl_version=None,
assert_hostname=None,
assert_fingerprint=None,
ca_cert_dir=None,
**conn_kw
):
HTTPConnectionPool.__init__(
self,
host,
port,
strict,
timeout,
maxsize,
block,
headers,
retries,
_proxy,
_proxy_headers,
**conn_kw
)
self.key_file = key_file
self.cert_file = cert_file
# 更新cert_reqs
self.cert_reqs = cert_reqs
self.key_password = key_password
self.ca_certs = ca_certs
self.ca_cert_dir = ca_cert_dir
self.ssl_version = ssl_version
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
def _prepare_conn(self, conn):
"""
Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
and establish the tunnel if proxy is used.
"""
if isinstance(conn, VerifiedHTTPSConnection):
# 调用创建ssl上下文变量
conn.set_cert(
key_file=self.key_file,
key_password=self.key_password,
cert_file=self.cert_file,
cert_reqs=self.cert_reqs,
ca_certs=self.ca_certs,
ca_cert_dir=self.ca_cert_dir,
assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint,
)
conn.ssl_version = self.ssl_version
return conn
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connection.py
def set_cert(
self,
key_file=None,
cert_file=None,
cert_reqs=None,
key_password=None,
ca_certs=None,
assert_hostname=None,
assert_fingerprint=None,
ca_cert_dir=None,
ca_cert_data=None,
):
"""
This method should only be called once, before the connection is used.
"""
# 如果没有传cert_reqs 会根据上下文获取，因为没有创建上下文，所以会用默认的VerifyMode.CERT_REQUIRED
if cert_reqs is None:
if self.ssl_context is not None:
cert_reqs = self.ssl_context.verify_mode
else:
cert_reqs = resolve_cert_reqs(None)
self.key_file = key_file
self.cert_file = cert_file
# 赋值，下边调用connect时会使用
self.cert_reqs = cert_reqs
def connect(self):
# Add certificate verification
conn = self._new_conn()
hostname = self.host
tls_in_tls = False
if self._is_using_tunnel():
if self.tls_in_tls_required:
conn = self._connect_tls_proxy(hostname, conn)
tls_in_tls = True
self.sock = conn
# Calls self._set_hostport(), so self.host is
# self._tunnel_host below.
self._tunnel()
# Mark this connection as not reusable
self.auto_open = 0
# Override the host with the one we're requesting data from.
hostname = self._tunnel_host
server_hostname = hostname
if self.server_hostname is not None:
server_hostname = self.server_hostname
is_time_off = datetime.date.today() < RECENT_DATE
if is_time_off:
warnings.warn(
(
"System time is way off (before {0}). This will probably "
"lead to SSL verification errors"
).format(RECENT_DATE),
SystemTimeWarning,
)
# Wrap socket using verification with the root certs in
# trusted_root_certs
default_ssl_context = False
# 如果没有上下文变量，则会根据创建默认的上下文变量，导致出现验证
if self.ssl_context is None:
default_ssl_context = True
self.ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(self.ssl_version),
cert_reqs=resolve_cert_reqs(self.cert_reqs),
)
context = self.ssl_context
context.verify_mode = resolve_cert_reqs(self.cert_reqs)
2、elasticsearch Python 8 直接使用verify_certs这个变量进行控制，即使传参时传了不需要验证的上下文，他自己还会根据verify_certs参数，如果它是true就会更新对应的数据，False时，直接创建一个不需要验证的上下文，连接的时候把这个上下文传递过去，8版本的在一开始transport传参时就进行了判断创建。
C:\Users\admin\Desktop\second\venv\Lib\site-packages\elastic_transport_node_http_urllib3.py
C:\Users\admin\Desktop\second\venv\Lib\site-packages\elastic_transport_node_http_urllib3.py
class Urllib3HttpNode(BaseNode):
def __init__(self, config: NodeConfig):
'''省略'''
if config.scheme == "https":
pool_class = HTTPSConnectionPool
# 根据你的配置创建上下文，如果你传了上下文直接返回你自己的上下文，但是下边会根据verify_certs的值给你更新掉
ssl_context = ssl_context_from_node_config(config)
kw["ssl_context"] = ssl_context
if config.ssl_assert_hostname and config.ssl_assert_fingerprint:
raise ValueError(
"Can't specify both 'ssl_assert_hostname' and 'ssl_assert_fingerprint'"
)
elif config.ssl_assert_fingerprint:
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
kw.update(
{
"assert_fingerprint": config.ssl_assert_fingerprint,
"assert_hostname": False,
"cert_reqs": "CERT_NONE",
}
)
else:
kw["assert_hostname"] = config.ssl_assert_hostname
# Convert all sentinel values to their actual default
# values if not using an SSLContext.
ca_certs = (
DEFAULT_CA_CERTS if config.ca_certs is None else config.ca_certs
)
# 就在这里根据verify_certs把你的上下文更新掉，cert_reqs为CERT_NONE即不需要验证
if config.verify_certs:
if not ca_certs:
raise ValueError(
"Root certificates are missing for certificate "
"validation. Either pass them in using the ca_certs parameter or "
"install certifi to use it automatically."
)
kw.update(
{
"cert_reqs": "CERT_REQUIRED",
"ca_certs": ca_certs,
"cert_file": config.client_cert,
"key_file": config.client_key,
}
)
else:
kw["cert_reqs"] = "CERT_NONE"
'''省略'''
下边的代码是6.1.3版本的
大致看了一眼代码：
1、
client = Elasticsearch(hosts="https://ip:9200/", http_auth=('用户名', '密码'), ssl=True, verify_certs=False)
1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\client_init_.py
1.1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\transport.py
1.1.1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\connection\http_urllib3.py
1.1.1.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\util\request.py
设置的用户名密码就是在这里被翻译的
1.1.1.2、
和上边1.1.1.1函数在同一个函数中
if ssl_context and self.use_ssl:
pool_class = urllib3.HTTPSConnectionPool
kw.update({
'assert_fingerprint': ssl_assert_fingerprint,
'ssl_context': ssl_context,
})
elif self.use_ssl:
pool_class = urllib3.HTTPSConnectionPool
kw.update({
'ssl_version': ssl_version,
'assert_hostname': ssl_assert_hostname,
'assert_fingerprint': ssl_assert_fingerprint,
})
# If `verify_certs` is sentinal value, default `verify_certs` to `True`
if verify_certs is VERIFY_CERTS_DEFAULT: # 不验证的话直接跳到else，不处理
verify_certs = True
ca_certs = CA_CERTS if ca_certs is None else ca_certs
if verify_certs:
if not ca_certs:
raise ImproperlyConfigured("Root certificates are missing for certificate "
"validation. Either pass them in using the ca_certs parameter or "
"install certifi to use it automatically.")
kw.update({
'cert_reqs': 'CERT_REQUIRED',
'ca_certs': ca_certs,
'cert_file': client_cert,
'key_file': client_key,
})
else:
warnings.warn(
'Connecting to %s using SSL with verify_certs=False is insecure.' % host)
1.2、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\transport.py
1.2.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\elasticsearch\connection\http_urllib3.py
1.2.1.1、
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py
1.2.1.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py
1.2.1.1.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connection.py
1.2.1.2、
1.2.1.2.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connectionpool.py
1.2.1.2.1.1
C:\Users\admin\AppData\Local\Programs\Python\Python39\Lib\site-packages\urllib3\connection.py
1.2.1.2.1.1.1
