Tungsten Fabric(5):对接vMX填坑
Tungsten Fabric作为Juniper开源的控制器，自然支持对接管控自家的交换机，主要是MX和QFX系列交换机。当然，对于绝大多数人来说，不可能因为要了解对接的机制就去真的购买Juniper的物理交换机，更现实的方法是使用MX的虚拟机版本vMX来做模拟实验。下面做的实验就是基于EVE-NG模拟平台，在CentOS虚拟机上做TF&k8s集成部署，然后启动vMX来做对接。
需要注意的是，本文着眼于能够对接并对vMX下发配置，其中很多方法其实是绕过了问题而并没有真正解决问题。
准备
- 后续对python文件的修改，都需要进入device-manager这个docker进行修改
[root@master01 ~]# docker ps | grep devi
001caca78cde hub.juniper.net/contrail-controller-config-devicemgr:1912-latest "/entrypoint.sh /usr…" 8 hours ago Up About an hour config_devicemgr_1
[root@master01 ~]#
[root@master01 ~]# docker exec -it 001caca78cde bash
(config-device-manager)[root@master01 /]$ cd /usr/lib/python2.7/site-packages/device_manager
(config-device-manager)[root@master01 /usr/lib/python2.7/site-packages/device_manager]$
(config-device-manager)[root@master01 /usr/lib/python2.7/site-packages/device_manager]$ ll | egrep "*.py$"
-rw-r--r--. 1 root root 0 Nov 18 18:30 __init__.py
-rw-r--r--. 1 root root 6409 Nov 18 18:30 ansible_base.py
-rw-r--r--. 1 root root 27640 Apr 19 23:40 ansible_conf.py
-rw-r--r--. 1 root root 69004 Nov 18 18:30 ansible_role_common.py
-rw-r--r--. 1 root root 1256 Nov 18 18:30 assisted_replicator_feature.py
-rw-r--r--. 1 root root 120643 Nov 18 18:30 db.py
-rw-r--r--. 1 root root 6626 Apr 19 22:25 device_conf.py
-rw-r--r--. 1 root root 32866 Nov 18 18:30 device_job_manager.py
-rw-r--r--. 1 root root 20691 Nov 18 18:30 device_manager.py
-rw-r--r--. 1 root root 11726 Nov 18 18:30 device_ztp_manager.py
-rw-r--r--. 1 root root 2054 Nov 18 18:30 dm_amqp.py
-rw-r--r--. 1 root root 6351 Nov 18 18:30 dm_server.py
-rw-r--r--. 1 root root 11173 Nov 18 18:30 dm_server_args.py
-rw-r--r--. 1 root root 20935 Nov 18 18:30 dm_utils.py
-rw-r--r--. 1 root root 76077 Apr 19 22:03 e2_conf.py
-rw-r--r--. 1 root root 3397 Nov 18 18:30 e2_services_info.py
-rw-r--r--. 1 root root 9484 Nov 18 18:30 fabric_manager.py
-rw-r--r--. 1 root root 10850 Nov 18 18:30 feature_base.py
-rw-r--r--. 1 root root 1309 Nov 18 18:30 imports.py
-rw-r--r--. 1 root root 7797 Nov 18 18:30 job_handler.py
-rw-r--r--. 1 root root 26029 Apr 20 02:49 juniper_conf.py
-rw-r--r--. 1 root root 6247 Nov 18 18:30 l2_gateway_feature.py
-rw-r--r--. 1 root root 3367 Nov 18 18:30 l3_gateway_feature.py
-rw-r--r--. 1 root root 12661 Nov 18 18:30 logger.py
-rw-r--r--. 1 root root 39551 Nov 18 18:30 mx_conf.py
-rw-r--r--. 1 root root 4164 Nov 18 18:30 opserver_util.py
-rw-r--r--. 1 root root 6568 Nov 18 18:30 overlay_bgp_feature.py
-rw-r--r--. 1 root root 1082 Nov 18 18:30 overlay_conf.py
-rw-r--r--. 1 root root 16762 Nov 18 18:30 pnf_conf.py
-rw-r--r--. 1 root root 2029 Nov 18 18:30 qfx_10k.py
-rw-r--r--. 1 root root 1612 Nov 18 18:30 qfx_5k.py
-rw-r--r--. 1 root root 42072 Nov 18 18:30 qfx_conf.py
-rw-r--r--. 1 root root 4932 Nov 18 18:30 storm_control_feature.py
-rw-r--r--. 1 root root 7297 Nov 18 18:30 telemetry_feature.py
-rw-r--r--. 1 root root 5068 Nov 18 18:30 underlay_ip_clos_feature.py
-rw-r--r--. 1 root root 5485 Nov 18 18:30 vn_interconnect_feature.py
(config-device-manager)[root@master01 /usr/lib/python2.7/site-packages/device_manager]$
- 修改前记得先备份，例如
# cp juniper_conf.py juniper_conf.py_bak
- 修改后需要重启device-manager这个docker，修改才会生效
[root@master01 ~]# docker restart 001caca78cde
001caca78cde
- log打印最好一直开着，默认只有WARNNING和ERROR的信息会打印出来
[root@master01 ~]# tailf /var/log/contrail/contrail-device-manager.log
- 根据打印信息找到代码位置，是最便捷的定位方法
- 如果要自己向py文件中添加打印信息，下面是个例子
self._logger.warning(">>password is %s" % self.user_creds[password])
- 如果发现修改后docker不停地自动重启，那就抓紧时间进入docker后，将之前修改的py文件恢复，例如
# cd /usr/lib/python2.7/site-packages/device_manager
# cp juniper_conf.py_bak juniper_conf.py
坑一：
- 描述：TF5.1.0版本默认对接物理交换机的功能是关闭的
- 解决方法：参考这里，需要手动添加一项，建议接着删除所有contrail相关的容器后重建。
[root@master01 ~]# cat /etc/contrail/common_config.env
KUBERNETES_API_SERVER=192.168.122.177
TTY=True
ANALYTICSDB_NODES=192.168.122.177
ANALYTICS_SNMP_ENABLE=False
STDIN_OPEN=True
WEBUI_NODES=192.168.122.177
CONTROLLER_NODES=192.168.122.177
KUBERNETES_API_NODES=192.168.122.177
ANALYTICS_NODES=192.168.122.177
ANALYTICSDB_ENABLE=True
CONFIGDB_NODES=192.168.122.177
ANALYTICS_ALARM_ENABLE=False
CLOUD_ORCHESTRATOR=kubernetes
CONTROL_NODES=192.168.122.177
KUBEMANAGER_NODES=192.168.122.177
CONFIG_NODES=192.168.122.177
CONTRAIL_VERSION=1912-latest
DEVICE_MANAGER__DEFAULTS__push_mode=0 #<<<<<<<新加此行
[root@master01 ~]#
[root@master01 ~]# docker-compose -f /etc/contrail/config/docker-compose.yaml down
/usr/lib/python2.7/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.24.3) or chardet (2.2.1) doesn't match a supported version!
RequestsDependencyWarning)
Stopping config_devicemgr_1 ... done
Stopping config_svcmonitor_1 ... done
Stopping config_api_1 ... done
Stopping config_schema_1 ... done
Stopping config_nodemgr_1 ... done
Stopping config_dnsmasq_1 ... done
Stopping config_stats_1 ... done
Removing config_devicemgr_1 ... done
Removing config_svcmonitor_1 ... done
Removing config_api_1 ... done
Removing config_schema_1 ... done
Removing config_nodemgr_1 ... done
Removing config_dnsmasq_1 ... done
Removing config_stats_1 ... done
Removing config_node-init_1 ... done
[root@master01 ~]#
[root@master01 ~]# docker-compose -f /etc/contrail/config/docker-compose.yaml up -d
/usr/lib/python2.7/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.24.3) or chardet (2.2.1) doesn't match a supported version!
RequestsDependencyWarning)
Creating config_node-init_1 ... done
Creating config_svcmonitor_1 ... done
Creating config_schema_1 ... done
Creating config_api_1 ... done
Creating config_nodemgr_1 ... done
Creating config_stats_1 ... done
Creating config_dnsmasq_1 ... done
Creating config_devicemgr_1 ... done
[root@master01 ~]#
坑二：
- 描述：TF的netconf session会去对接交换机的接口值已经被写死(juniper_conf.py),所以在web录入的时候service port填的值不会生效
def device_connect(self):
if not self._nc_manager or not self.is_connected():
try:
self._nc_manager = manager.connect(host=self.management_ip, port=22,
- 解决方法：物理交换机的netconf port设置为22
[edit]
netops@vMX-3# set system services netconf ssh port 22 ，才好对接
坑三：
- 描述：代码传入的netconf password是密文，但是ncclient只能接受密码的原文，导致
04/19/2020 08:00:53 PM [contrail-device-manager] [ERROR]: could not establish netconf session with router 192.168.122.103: AuthenticationException('Authentication failed.',)
例如我设置的密码是"Test123"，但是其实传的是
- 解决方法(juniper_conf.py)：是的，我就直接赋值了
def device_connect(self):
if not self._nc_manager or not self.is_connected():
self._logger.warning(">>keys are %s" % self.user_creds.keys())
self._logger.warning(">>type of password is %s" % type(self.user_creds['password']))
self._logger.warning(">>password is %s" % self.user_creds['password'])
try:
self._nc_manager = manager.connect(host=self.management_ip, port=22,
username=self.user_creds['username'],
#password=self.user_creds['password'],
password="Test123",
坑四：
- 描述：netconf对接成功后，TF会取回device model信息，并判断是否是mx或vmx，但是我用的模拟器返回的是olive
netops@vMX-3> show version
Hostname: vMX-3
Model: olive
Junos: 18.2R2.6
因此报错
04/20/2020 09:02:06 PM [contrail-device-manager] [ERROR]: product model mismatch: device model = olive, plugin mode = ['mx', 'vmx']
- 解决方法：device_conf.py直接给model赋值
def validate_device(self):
if not self.device_config:
self.device_config = self.device_get()
if not self.device_config:
self.device_config = {}
return False
#model = self.device_config.get('product-model')
model = 'mx'
坑五：
- 描述：添加BGP Router后，配置无法下发成功。明明输入的是64512，结果识别为"b’64512’"
04/19/2020 09:25:38 PM [contrail-device-manager] [ERROR]: Router 192.168.122.103: error: as_number: 'b'64512'': Use format 'x' or 'x.y' to specify number 'x' (Range 1 .. 4294967295) or number with higher 2 bytes 'x' (Range 0 .. 65535) ay
error: syntax error, expecting or
将conf_str打印出来self._logger.warning(">>config_str is %s" % config_str)
/* Contrail Generated Group Config */
__contrail__
/* Global Routing Options */
3.3.3.3
3.3.3.3
b'64512'
bgp.rtarget.0
inet.0
/* Protocols Configuration */
/* overlay_bgp: BGP Router: qfx5100-48t-1, UUID: f0afa9d3-d388-45ea-b6e2-c5319e9664e0 */
b'_contrail_asn-64512'
internal
3.3.3.3
90
/* overlay_bgp: BGP Router: master01.local, UUID: b04bc39c-57ea-4f0e-b04a-3d2737ce242f */
192.168.122.177
64512
/* Policy Options */
_contrail_switch_policy_
b'target:64512:1'
lo0.0
__contrail__
- 解决方法：经过大神指点，将contrail-controller-config-devicemgr镜像从1912-latest替换为1910-laset
- 在我的deployer节点操作
[root@deployer ~]# docker pull opencontrailnightly/contrail-controller-config-devicemgr:1910-latest
[root@deployer ~]# docker tag opencontrailnightly/contrail-controller-config-devicemgr:1910-latest 192.168.122.211/contrail-controller-config-devicemgr:1910-latest
[root@deployer ~]# docker push 192.168.122.211/contrail-controller-config-devicemgr:1910-latest
[root@deployer ~]#
[root@deployer ~]# curl -X GET http://localhost/v2/contrail-controller-config-devicemgr/tags/list --cacert /etc/docker/certs.d/dockershare:2333/ca.crt | python -m json.tool
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 85 100 85 0 0 1072 0 --:--:-- --:--:-- --:--:-- 1089
{
"name": "contrail-controller-config-devicemgr",
"tags": [
"1912-latest",
"1910-latest"
]
}
[root@deployer ~]#
- master01节点拉取(hosts文件里面已经将hub.juniper.net设置为deployer IP)
[root@master01 ~]# ping hub.juniper.net
PING deployer (192.168.122.211) 56(84) bytes of data.
64 bytes from deployer (192.168.122.211): icmp_seq=1 ttl=64 time=1.47 ms
^C
--- deployer ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.472/1.472/1.472/0.000 ms
[root@master01 ~]#
[root@master01 ~]# docker pull hub.juniper.net/contrail-controller-config-devicemgr:1910-latest
[root@master01 ~]#
[root@master01 ~]# docker image list | grep devicemgr
hub.juniper.net/contrail-controller-config-devicemgr 1912-latest c08868a27a0a 5 months ago 772MB
hub.juniper.net/contrail-controller-config-devicemgr 1910-latest a1b3ab402efc 5 months ago 862MB
[root@master01 ~]#
- 修改/etc/contrail/config/docker-compose.yaml文件，将devicemgr的镜像版本从1912-latest改为1910-latest
devicemgr:
image: "hub.juniper.net/contrail-controller-config-devicemgr:1910-latest"
- 将contrail相关的容器删除然后重建
[root@master01 config]# docker-compose -f /etc/contrail/config/docker-compose.yaml down
/usr/lib/python2.7/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.24.3) or chardet (2.2.1) doesn't match a supported version!
RequestsDependencyWarning)
Stopping config_devicemgr_1 ... done
Stopping config_dnsmasq_1 ... done
Stopping config_nodemgr_1 ... done
Stopping config_schema_1 ... done
Stopping config_svcmonitor_1 ... done
Stopping config_stats_1 ... done
Stopping config_api_1 ... done
Removing config_devicemgr_1 ... done
Removing config_dnsmasq_1 ... done
Removing config_nodemgr_1 ... done
Removing config_schema_1 ... done
Removing config_svcmonitor_1 ... done
Removing config_stats_1 ... done
Removing config_api_1 ... done
Removing config_node-init_1 ... done
[root@master01 config]#
[root@master01 config]# docker-compose -f /etc/contrail/config/docker-compose.yaml up -d
/usr/lib/python2.7/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.24.3) or chardet (2.2.1) doesn't match a supported version!
RequestsDependencyWarning)
Creating config_node-init_1 ... done
Creating config_schema_1 ... done
Creating config_api_1 ... done
Creating config_svcmonitor_1 ... done
Creating config_nodemgr_1 ... done
Creating config_dnsmasq_1 ... done
Creating config_stats_1 ... done
Creating config_devicemgr_1 ... done
[root@master01 config]#
- 前面提到的坑1-4，需要在新的devicemgr中重新修改一遍
验证
- Physical Router已经添加(role为None)
- 添加BGP Router
- 查看vMX上新增配置
netops@vMX-3> show configuration | compare rollback 2
[edit]
+ groups {
+ /* Contrail Generated Group Config */
+ __contrail__ {
+ routing-options {
+ /* Global Routing Options */
+ router-id 3.3.3.3;
+ route-distinguisher-id 3.3.3.3;
+ autonomous-system 64512;
+ dynamic-tunnels {
+ _contrail_asn-64512 {
+ source-address 3.3.3.3;
+ gre;
+ destination-networks {
+ /* IP Fabric Subnet */
+ 192.168.122.0/24;
+ /* BGP Router : MX3 */
+ 3.3.3.3/32;
+ /* BGP Router : master01.local */
+ 192.168.122.177/32;
+ }
+ }
+ }
+ }
+ protocols {
+ /* Protocols Configuration */
+ bgp {
+ /* overlay_bgp: BGP Router: MX3, UUID: 2cf712e4-a3f2-4978-82a3-88c993754202 */
+ group _contrail_asn-64512 {
+ type internal;
+ local-address 3.3.3.3;
+ hold-time 90;
+ family inet- {
+ unicast;
+ }
+ family e {
+ signaling;
+ }
+ family route-target;
+ export _contrail_ibgp_export_policy;
+ /* overlay_bgp: BGP Router: master01.local, UUID: b04bc39c-57ea-4f0e-b04a-3d2737ce242f */
+ neighbor 192.168.122.177 {
+ peer-as 64512;
+ }
+ }
+ }
+ }
+ policy-options {
+ /* iBGP Export Policy */
+ policy-statement _contrail_ibgp_export_policy {
+ term inet- {
+ from family inet-;
+ then {
+ next-hop self;
+ }
+ }
+ }
+ }
+ }
+ }
+ apply-groups __contrail__;
netops@vMX-3>