如何实现OpenStack STT隧道(by quqi99)
作者：张华 发表于：2016-05-21
版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明
( http://blog.csdn.net/quqi99 )
代码
VMWare NSX使用OVS STT隧道，OVS是支持STT隧道，但是Neutron还不支持。所以我rebase了社区一个废弃很久的代码想测试一下TSO/GSO对其的影响（冲突很多，改动还有点大，代码见附件）。
机器
node1, duduo.seg, controller node
- eth0: 10.230.56.14, 1000Mb/s, br-ex
- eth6: 10.230.56.15, 10000Mb/s, out-off-band management
- eth7: 192.168.122.2, 10000Mb/s, br-phy
sudo ovs-vsctl -- --may-exist add-port br-ex eth0
sudo ifconfig eth0 0.0.0.0 up
sudo ifconfig br-ex 10.230.56.233/21
sudo ovs-vsctl -- --may-exist add-br br-phy
sudo ovs-vsctl -- --may-exist add-port br-phy eth7
sudo ifconfig eth7 0.0.0.0 up
sudo ifconfig br-phy 192.168.122.2/24
- eth0: 10.230.56.21, 1000Mb/s, out-off-band management
- eth4: 192.168.122.3, 10000Mb/s, br-phy
sudo ovs-vsctl -- --may-exist add-port br-phy eth4
sudo ifconfig eth4 0.0.0.0 up
sudo ifconfig br-phy 192.168.122.3/24
Devstack
sudo mkdir -p /bak/openstack && sudo chown -R ubuntu:root /bak/
sudo mkdir -p /opt/stack && sudo chown -R ubuntu:root /opt/stack
cd /bak/openstack/ && git clone https://github.com/openstack-dev/devstack.git
这两台机器位于内网，故当要下载东西时需要临时添加代理（export http_proxy=http://squid.internal:3128 && export https_proxy=http://squid.internal:3128），下载完毕应立即取消代理(unset http_proxy && uset https_proxy)。所以./stack.sh需运行两遍，一遍加代理运行访问互联网下载完该下载的东西，第二遍设置去掉代理（一定要去掉代理，不然openstack代理中的http访问全会出一些与此不相关的错）及设置OFFLINE=True后再运行一遍。
locarc文件内容如下:
OFFLINE=True unset http_proxy unset https_proxy #export http_proxy=http://squid.internal:3128 #export https_proxy=http://squid.internal:3128 PUBLIC_INTERFACE=eth7 #计算节点改为eth4 HOST_IP=192.168.122.2 #计算节点改为192.168.122.3 SERVICE_HOST=192.168.122.2 #计算节点改为192.168.122.2 DEST=/bak/openstack #sudo apt-get install openvswitch-switch qemu-kvm libvirt-bin #sudo virsh net-destroy default #两机器由MaaS申请，MaaS与Qemu都用了网段192.168.122.0/24冲突了影响路由造成 会两机器网络不通 sudo ovs-vsctl -- --may-exist add-br br-phy #先建好br-phy，避免devstack创建网桥时将IP从eth7移到br-phy造成 网络混乱 #sudo ovs-vsctl -- --may-exist add-port br-phy eth7 -- set interface eth7 type=internal #使用type=internal时路由在eth7上会造成"ovs-ofctl dump-flows br-phy“命令失败 sudo ovs-vsctl -- --may-exist add-port br-phy eth7 sudo ifconfig eth7 0.0.0.0 up sudo ifconfig br-phy 192.168.122.2/24 sudo ovs-vsctl -- --may-exist add-br br-ex sudo ovs-vsctl -- --may-exist add-port br-ex eth0 sudo ifconfig eth0 0.0.0.0 up sudo ifconfig br-ex 10.230.56.233/21 sleep 5 #ENABLED_SERVICES=n-cpu,rabbit,neutron,q-agt ENABLED_SERVICES=rabbit,mysql,key,g-api,g-reg ENABLED_SERVICES+=,n-api,n-crt,n-obj,n-cpu,n-cond,n-sch,n-cauth,n-novnc ENABLED_SERVICES+=,q-svc,q-agt,q-dhcp,q-l3,q-meta,neutron Q_USE_ROOTWRAP=True Q_USE_ROOTWRAP_DAEMON=False NOVA_VNC_ENABLED=True Q_ML2_TENANT_NETWORK_TYPE=flat,vlan,gre,vxlan,stt Q_ML2_PLUGIN_TYPE_DRIVERS=flat,vlan,gre,vxlan,stt #我们没有使用provider network(即flat, vlan物理网络）去自动创建网络，我们将手工创建，所以需要修改devstack>代码 #Q_USE_PROVIDER_NETWORKING=True #FIXED_RANGE="10.230.56.0/24" #NETWORK_GATEWAY=10.230.56.1 #PROVIDER_PROVIDER_SUBNET_NAMESUBNET_NAME="ex_net" #PROVIDER_NETWORK_TYPE="flat" GIT_BASE="https://git.openstack.org" #因为两台机器在内网中需要添加http_proxy代理 MYSQL_HOST=$SERVICE_HOST RABBIT_HOST=$SERVICE_HOST RABBIT_HEARTBEAT_TIMEOUT_THRESHOLD=120 #此参数和kombu_reconnect_delay=3.0一样重要是否rabbitmq连接总断开 GLANCE_HOSTPORT=$SERVICE_HOST:9292 Q_HOST=$SERVICE_HOST FIXED_RANGE=10.0.1.0/24 #113 -- 118, http://jodies.de/ipcalc FLOATING_RANGE=10.230.56.0/21 Q_FLOATING_ALLOCATION_POOL=start=10.230.56.100,end=10.230.56.104 PUBLIC_NETWORK_GATEWAY=10.230.56.1 NETWORK_GATEWAY=10.0.1.1 #若只有一块网卡，公网网络可与管理网络设置一致，可添加：Q_USE_PROVIDERNET_FOR_PUBLIC=True && PUBLIC_BRIDGE=br-phy PUBLIC_BRIDGE=br-ex # sudo ovs-vsctl -- --may-exist add-port br-ex eth0 -- set interface eth0 type=internal #事后采用type=internal添加不影响已有网络 OVS_PHYSICAL_BRIDGE=br-phy OVS_BRIDGE_MAPPINGS=physnet1:br-phy,public:br-ex IP_VERSION=4 DATABASE_USER=root DATABASE_PASSWORD=password ADMIN_PASSWORD=password SERVICE_PASSWORD=password RABBIT_PASSWORD=password SERVICE_TOKEN=ADMIN LOGFILE=$DEST/logs/stack.log ENABLE_DEBUG_LOG_LEVEL=False SYSLOG=False VERBOSE=True SCREEN_LOGDIR=$DEST/logs LOG_COLOR=False Q_USE_DEBUG_COMMAND=False APACHE_ENABLED_SERVICES+=keystone KEYSTONE_TOKEN_FORMAT=uuid USE_SSL=False disable_service tls-proxy IMAGE_URLS="https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img" DOWNLOAD_DEFAULT_IMAGES=False #IPSEC_PACKAGE=strongswan # Neutron Agent configuration Q_USE_NAMESPACE=True Q_ALLOW_OVERLAPPING_IP=True # Neutron ML2 configuration Q_PLUGIN=ml2 #Q_DVR_MODE=dvr_snat Q_DVR_MODE=legacy # VLAN ENABLE_TENANT_VLANS=True ML2_VLAN_RANGES=physnet1:1000:2999,physnet2:1000:2999 # GRE ENABLE_TENANT_TUNNELS=True TENANT_TUNNEL_RANGE=1000:2000 Q_ML2_PLUGIN_VXLAN_TYPE_OPTIONS=(vni_ranges=1000:2000) # Avanced ML2 configuration Q_AGENT=openvswitch Q_ML2_PLUGIN_MECHANISM_DRIVERS=openvswitch,l2population因为我不想devstack去创建网络（我想手工创建），我也不想devstack修改我已经创建好的br-phy网卡，所以需要定制devstack代码：
diff --git a/files/apts b/files/apts
deleted file mode 120000
index ef926de..0000000
--- a/files/apts
+++ /dev/null
@@ -1 +0,0 @@
-debs/
\ No newline at end of file
diff --git a/files/debs/openvswitch b/files/debs/openvswitch
index 4c0af4a..f99d0ef 100644
--- a/files/debs/openvswitch
+++ b/files/debs/openvswitch
@@ -1,3 +1,2 @@
fakeroot
make
-openvswitch-switch
diff --git a/lib/neutron b/lib/neutron
index ba26c5f..06475e7 100644
--- a/lib/neutron
+++ b/lib/neutron
@@ -489,7 +489,8 @@ function _set_config_files {
function cleanup_neutron {
if is_neutron_legacy_enabled; then
# Call back to old function
- cleanup_mutnauq "$@"
+ #cleanup_mutnauq "$@"
+ echo 'skip to move route from eth0 to br-phy'
else
cleanup_neutron_new "$@"
fi
diff --git a/lib/neutron_plugins/openvswitch_agent b/lib/neutron_plugins/openvswitch_agent
index 69e38f4..f9a4510 100644
--- a/lib/neutron_plugins/openvswitch_agent
+++ b/lib/neutron_plugins/openvswitch_agent
@@ -20,7 +20,8 @@ function neutron_plugin_create_nova_conf {
}
function neutron_plugin_install_agent_packages {
- _neutron_ovs_base_install_agent_packages
+ #_neutron_ovs_base_install_agent_packages
+ echo 'skip install ovs package'
}
function neutron_plugin_configure_debug_command {
diff --git a/lib/neutron_plugins/services/l3 b/lib/neutron_plugins/services/l3
index 350aed8..dab8432 100644
--- a/lib/neutron_plugins/services/l3
+++ b/lib/neutron_plugins/services/l3
@@ -101,10 +101,11 @@ function _configure_neutron_l3_agent {
neutron_plugin_configure_l3_agent
- _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True False "inet"
+ #_move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" True False "inet"
if [[ $(ip -f inet6 a s dev "$PUBLIC_INTERFACE" | grep -c 'global') != 0 ]]; then
- _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False False "inet6"
+ # _move_neutron_addresses_route "$PUBLIC_INTERFACE" "$OVS_PHYSICAL_BRIDGE" False False "inet6"
+ echo 'avoid to move ip'
fi
}
@@ -348,8 +349,8 @@ function _neutron_configure_router_v6 {
local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}
# Configure interface for public bridge
- sudo ip -6 addr add $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
- sudo ip -6 route replace $FIXED_RANGE_V6 via $IPV6_ROUTER_GW_IP dev $ext_gw_interface
+ #sudo ip -6 addr add $ipv6_ext_gw_ip/$ipv6_cidr_len dev $ext_gw_interface
+ #sudo ip -6 route replace $FIXED_RANGE_V6 via $IPV6_ROUTER_GW_IP dev $ext_gw_interface
fi
_neutron_set_router_id
fi
diff --git a/stack.sh b/stack.sh
index 6fbb0be..5d15ed7 100755
--- a/stack.sh
+++ b/stack.sh
@@ -1256,7 +1256,7 @@ fi
# Once neutron agents are started setup initial network elements
if is_service_enabled q-svc && [[ "$NEUTRON_CREATE_INITIAL_NETWORKS" == "True" ]]; then
echo_summary "Creating initial neutron network elements"
- create_neutron_initial_network
+ #create_neutron_initial_network
fi
if is_service_enabled nova; then
diff --git a/tools/install_pip.sh b/tools/install_pip.sh
index 1267699..eca967d 100755
--- a/tools/install_pip.sh
+++ b/tools/install_pip.sh
@@ -76,7 +76,7 @@ function install_get_pip {
die $LINENO "Download of get-pip.py failed"
touch $LOCAL_PIP.downloaded
fi
- sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
+ #sudo -H -E python $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
if python3_enabled; then
sudo -H -E python${PYTHON3_VERSION} $LOCAL_PIP -c $TOOLS_DIR/cap-pip.txt
fi
升级stt代码与数据库
cd ../neutron && patch -p1 < stt.diff
sudo python setup.py develop #生成STT_TYPE_Driver
screen -x stack
$ grep 'stt' /etc/neutron/plugins/ml2/ml2_conf.ini
tenant_network_types = flat,vlan,gre,vxlan,stt
type_drivers = flat,vlan,gre,vxlan,stt
[agent]
tunnel_types = stt
下列方法升级数据库后在screen中重启neutron-server与neutron-ovs-agent两个服务
#http://docs.openstack.org/developer/neutron/devref/alembic_migrations.html
neutron-db-manage current --verbose
neutron-db-manage upgrade heads
创建网络 & 测试步骤
export OS_USERNAME=admin
export OS_PASSWORD=password
export OS_TENANT_NAME=demo
export OS_AUTH_URL=http://192.168.122.3:5000/v2.0
export OS_AUTH_STRATEGY=keystone
neutron net-create net-stt --provider:network_type stt --provider:segmentation_id 1012
neutron subnet-create --allocation-pool start=10.0.1.22,end=10.0.1.122 --gateway 10.0.1.1 net-stt 10.0.1.0/24 --enable_dhcp=True --name subnet-stt
neutron net-create public -- --router:external=True --provider:network_type flat --provider:physical_network public
neutron subnet-create --allocation-pool start=10.230.56.100,end=10.230.56.104 --gateway 10.230.56.1 public 10.230.56.100/21 --enable_dhcp=False --name public-subnet
neutron router-create router1
EXT_NET_ID=$(neutron net-list |grep ' public ' |awk '{print $2}')
ROUTER_ID=$(neutron router-list |grep ' router1 ' |awk '{print $2}')
SUBNET_ID=$(neutron subnet-list |grep '10.0.1.0/24' |awk '{print $2}')
neutron router-interface-add $ROUTER_ID $SUBNET_ID
neutron router-gateway-set $ROUTER_ID $EXT_NET_ID
sudo rabbitmqctl status
neutron agent-list
#nova service-disable duduo nova-compute
nova service-list
glance image-list
nova flavor-list
neutron net-list
nova secgroup-add-rule default tcp 22 22 0.0.0.0/0
nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0
neutron security-group-rule-create --direction ingress --protocol TCP --port-range-min 8000 --port-range-max 8000 default
neutron security-group-rule-create --direction ingress --protocol TCP --port-range-min 5001 --port-range-max 5001 default
nova keypair-add --pub-key ~/.ssh/id_rsa.pub mykey
NET_ID=$(neutron net-list |grep ' net-stt ' |awk '{print $2}')
nova boot --poll --key-name mykey --image trusty-server-cloudimg-amd64-disk1 --flavor 2 --nic net-id=$NET_ID i1
nova floating-ip-create
nova floating-ip-associate i1 10.230.56.103
ssh -o StrictHostKeyChecking=no -i mykey [email protected] -v
nova service-list
nova hypervisor-list
nova boot --poll --key-name mykey --image trusty-server-cloudimg-amd64-disk1 --flavor 2 --nic net-id=$NET_ID --availability-zone nova:voltorb i2
nova floating-ip-create
nova floating-ip-associate i2 10.230.56.104
ssh -o StrictHostKeyChecking=no -i mykey [email protected] -v
ubuntu@duduo:~$ nova list
+--------------------------------------+------+--------+------------+-------------+----------------------------------+
| ID | Name | Status | Task State | Power State | Networks |
+--------------------------------------+------+--------+------------+-------------+----------------------------------+
| 5e9bcd85-6ac7-4ec7-9f5f-89a2ed3c5fb4 | i1 | ACTIVE | - | Running | net-stt=10.0.1.32, 10.230.56.103 |
| ed84c40a-ecde-4e77-ae4f-a353ab72e78e | i2 | ACTIVE | - | Running | net-stt=10.0.1.30, 10.230.56.104 |
+--------------------------------------+------+--------+------------+-------------+----------------------------------+
遇到的问题
kombu_reconnect_delay=3.0
heartbeat_timeout_threshold=120
ovs-ofctl: br-phy: failed to connect to socket (Connection reset by peer)
ubuntu@duduo:/bak/openstack/devstack$ sudo ovs-vsctl del-port br-phy eth7
ubuntu@duduo:/bak/openstack/devstack$ sudo ovs-ofctl dump-flows br-phy
NXST_FLOW reply (xid=0x4):
cookie=0x0, duration=11533.961s, table=0, n_packets=20103, n_bytes=6910304, idle_age=3, priority=0 actions=NORMAL
ubuntu@duduo:/bak/openstack/devstack$ sudo ovs-ofctl dump-flows br-phy
NXST_FLOW reply (xid=0x4):
cookie=0x0, duration=11584.170s, table=0, n_packets=20125, n_bytes=6912276, idle_age=0, priority=0 actions=NORMAL
ubuntu@duduo:/bak/openstack/devstack$ sudo ovs-vsctl del-port br-phy eth7
ubuntu@duduo:/bak/openstack/devstack$ sudo ovs-vsctl -- --may-exist add-port br-phy eth7 -- set interface eth7 type=internal
ubuntu@duduo:/bak/openstack/devstack$ sudo ovs-ofctl dump-flows br-phy
ovs-ofctl: br-phy: failed to connect to socket (Connection reset by peer)
ubuntu@voltorb:/bak/openstack/devstack$ sudo brctl show
bridge name bridge id STP enabled interfaces
virbr0 8000.525400e658db yes virbr0-nic
ubuntu@voltorb:/bak/openstack/devstack$ sudo virsh net-destroy default
4, 创建虚机时调度那一块出现ERROR，多半是要将RetryFilter移除。
5, 创建虚机时无法从glance获取镜像报错”，Neutron也报一些“Connection timed out”的错，原因是内网机器下载使用了http_proxy，需要移除代理。
2016-05-20 12:16:24.074 444 ERROR nova.compute.manager [instance: efc19b8b-aeb5-4b49-ba89-4ba6b0696d34] HTTPException: ERROR: The requested URL could not be retrieved ERROR The requested URL could not be retrieved The following error was encountered while trying to retrieve the URL: http://192.168.122.3:9292/v1/images/b868b12f-0bf2-4b04-bead-03f9c5c52bb6 Connection to 192.168.122.3 failed. The system returned: (110) Connection timed out The remote host or network may be down. Please try the request again. Your cache administrator is webmaster. Generated Fri, 20 May 2016 12:16:24 GMT by shuppet.canonical.com (squid/2.7.STABLE7) (HTTP N/A)
NeutronClientException: <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html><head> <meta http-equiv="Content-Type" CONTENT="text/html; charset=utf-8"> <title>ERROR: The requested URL could not be retrieved</title> <style type="text/css"><!-- %l body :lang(fa) { direction: rtl; font-size: 100%; font-family: Tahoma, Roya, sans-serif; float: right; } :lang(he) { direction: rtl; } --></style> </head><body id=ERR_CONNECT_FAIL> <div id="titles"> <h1>ERROR</h1> <h2>The requested URL could not be retrieved</h2> </div> <hr> <div id="content"> <p>The following error was encountered while trying to retrieve the URL: <a href="http://192.168.122.3:9696/v2.0/extensions.json">http://192.168.122.3:9696/v2.0/extensions.json</a></p> <blockquote id="error"> <p><b>Connection to 192.168.122.3 failed.</b></p> </blockquote> <p id="sysmsg">The system returned: <i>(110) Connection timed out</i></p> <p>The remote host or network may be down. Please try the request again.</p> <p>Your cache administrator is <a href="mailto:webmaster%W">webmaster</a>.</p> <br> </div> <hr> <div id="footer"> <p>Generated Fri, 20 May 2016 12:16:23 GMT by shuppet.canonical.com (squid/2.7.STABLE7)</p> <!-- ERR_CONNECT_FAIL --> </div> </body></html>
Removing descriptor: 18
6, OVS STT在某些kernel版本上无法创建网卡，此问题还未解决
neutron-server.log
2016-05-20 14:37:17.068 25813 ERROR neutron.plugins.ml2.drivers.openvswitch.agent.ovs_neutron_agent [req-e7932a08-657c-48c9-a797-649e3019375b - -] Failed to set-up stt tunnel port to 192.168.122.3
/var/log/openvswitch/ovs-vswitchd.log
2016-05-20T14:37:16.941Z|00051|dpif|WARN|system@ovs-system: failed to add stt-c0a87a03 as port: Address family not supported by protocol
所以需要编译vport-stt.ko模块，方法如下：
#https://github.com/shettyg/ovn-docker/blob/master/docs/backend.md
sudo apt-get install -y autoconf libtool sparse openssl pkg-config make gcc libssl-dev git
git clone https://github.com/openvswitch/ovs.git
cd ovs
./boot.sh
./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --enable-ssl --with-linux=/lib/modules/`uname -r`/build
make -j8
sudo make install
sudo cp debian/openvswitch-switch.init /etc/init.d/openvswitch-switch
ubuntu@duduo:/bak/openstack/ovs$ sudo modinfo ./datapath/linux/openvswitch.ko |grep depends
depends: nf_conntrack,nf_defrag_ipv6,libcrc32c,gre,nf_defrag_ipv4
sudo modprobe gre
sudo modprobe libcrc32c
sudo modprobe nf_conntrack
sudo modprobe nf_defrag_ipv6
sudo modprobe nf_defrag_ipv4
sudo make modules_install
sudo rmmod -f openvswitch.ko #Or it will throw the error 'Unknown symbol in module'
sudo insmod ./datapath/linux/openvswitch.ko
sudo insmod ./datapath/linux/vport-stt.ko
sudo /etc/init.d/openvswitch-switch start
但是在ubuntu 16.04上报错如下，改到ubuntu 14.04上运行成功。
checking for kernel version... 4.4.8
configure: error: Linux kernel in /lib/modules/4.4.0-22-generic/build is version 4.4.8, but version newer than 4.3.x is not supported (please refer to the FAQ for advice)
Neutron STT ML2 Type Driver Code
diff --git a/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD b/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD
index cf26eb1..6a75db4 100644
--- a/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD
+++ b/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD
@@ -1 +1 @@
-30107ab6a3ee
+3abed5b82c57
diff --git a/neutron/db/migration/alembic_migrations/versions/newton/contract/3abed5b82c57_add_stt_type_driver.py b/neutron/db/migration/alembic_migrations/versions/newton/contract/3abed5b82c57_add_stt_type_driver.py
new file mode 100644
index 0000000..c305f6c
--- /dev/null
+++ b/neutron/db/migration/alembic_migrations/versions/newton/contract/3abed5b82c57_add_stt_type_driver.py
@@ -0,0 +1,49 @@
+# Copyright 2015 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+"""add_stt_type_driver
+
+Revision ID: 3abed5b82c57
+Revises: 30107ab6a3ee
+Create Date: 2016-05-19 12:52:40.546353
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3abed5b82c57'
+down_revision = '30107ab6a3ee'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.create_table(
+ 'ml2_stt_allocations',
+ sa.Column('stt_context_id', sa.Integer(),
+ autoincrement=False, nullable=False),
+ sa.Column('allocated', sa.Boolean(),
+ server_default=sa.sql.false(), nullable=False),
+ sa.PrimaryKeyConstraint('stt_context_id'),
+ )
+ op.create_index(op.f('ix_ml2_stt_allocations_allocated'),
+ 'ml2_stt_allocations', ['allocated'], unique=False)
+ op.create_table(
+ 'ml2_stt_endpoints',
+ sa.Column('ip_address', sa.String(length=64), nullable=False),
+ sa.Column('host', sa.String(length=255), nullable=True),
+ sa.PrimaryKeyConstraint('ip_address'),
+ sa.UniqueConstraint('host', name='unique_ml2_stt_endpoints0host'),
+ )
diff --git a/neutron/db/migration/models/head.py b/neutron/db/migration/models/head.py
index 6112dbf..2b336ac 100644
--- a/neutron/db/migration/models/head.py
+++ b/neutron/db/migration/models/head.py
@@ -54,6 +54,7 @@ from neutron.ipam.drivers.neutrondb_ipam import db_models # noqa
from neutron.plugins.ml2.drivers import type_flat # noqa
from neutron.plugins.ml2.drivers import type_geneve # noqa
from neutron.plugins.ml2.drivers import type_gre # noqa
+from neutron.plugins.ml2.drivers import type_stt # noqa
from neutron.plugins.ml2.drivers import type_vlan # noqa
from neutron.plugins.ml2.drivers import type_vxlan # noqa
from neutron.plugins.ml2 import models # noqa
diff --git a/neutron/plugins/common/constants.py b/neutron/plugins/common/constants.py
index 4b38514..2a6ced1 100644
--- a/neutron/plugins/common/constants.py
+++ b/neutron/plugins/common/constants.py
@@ -69,6 +69,7 @@ TYPE_GENEVE = 'geneve'
TYPE_GRE = 'gre'
TYPE_LOCAL = 'local'
TYPE_VXLAN = 'vxlan'
+TYPE_STT = 'stt'
TYPE_VLAN = 'vlan'
TYPE_NONE = 'none'
@@ -86,6 +87,10 @@ MAX_GENEVE_VNI = 2 ** 24 - 1
MIN_GRE_ID = 1
MAX_GRE_ID = 2 ** 32 - 1
+# For STT Tunnel
+MIN_STT_CONTEXT_ID = 1
+MAX_STT_CONTEXT_ID = 2 ** 64 - 1
+
# For VXLAN Tunnel
MIN_VXLAN_VNI = 1
MAX_VXLAN_VNI = 2 ** 24 - 1
@@ -95,3 +100,6 @@ VXLAN_UDP_PORT = 4789
GENEVE_ENCAP_MIN_OVERHEAD = 50
GRE_ENCAP_OVERHEAD = 42
VXLAN_ENCAP_OVERHEAD = 50
+# NOTE(arosen): STT does not need an ENCAP_OVERHEAD as it's TCP based
+# thus L2 fragmentation will not come into play. The packets will be
+# sized to the systems configured MTU.
diff --git a/neutron/plugins/ml2/config.py b/neutron/plugins/ml2/config.py
index 4e2e43d..f85a9f5 100644
--- a/neutron/plugins/ml2/config.py
+++ b/neutron/plugins/ml2/config.py
@@ -19,7 +19,8 @@ from neutron._i18n import _
ml2_opts = [
cfg.ListOpt('type_drivers',
- default=['local', 'flat', 'vlan', 'gre', 'vxlan', 'geneve'],
+ default=['local', 'flat', 'vlan', 'gre', 'vxlan', 'stt',
+ 'geneve'],
help=_("List of network type driver entrypoints to be loaded "
"from the neutron.ml2.type_drivers namespace.")),
cfg.ListOpt('tenant_network_types',
diff --git a/neutron/plugins/ml2/drivers/openvswitch/agent/common/constants.py b/neutron/plugins/ml2/drivers/openvswitch/agent/common/constants.py
index 96903d3..b996f39 100644
--- a/neutron/plugins/ml2/drivers/openvswitch/agent/common/constants.py
+++ b/neutron/plugins/ml2/drivers/openvswitch/agent/common/constants.py
@@ -33,7 +33,7 @@ NONEXISTENT_PEER = 'nonexistent-peer'
# The different types of tunnels
TUNNEL_NETWORK_TYPES = [p_const.TYPE_GRE, p_const.TYPE_VXLAN,
- p_const.TYPE_GENEVE]
+ p_const.TYPE_GENEVE, p_const.TYPE_STT]
### OpenFlow table IDs
@@ -75,6 +75,7 @@ DVR_PROCESS = 1
PATCH_LV_TO_TUN = 2
GRE_TUN_TO_LV = 3
VXLAN_TUN_TO_LV = 4
+STT_TUN_TO_LV = 5
GENEVE_TUN_TO_LV = 6
DVR_NOT_LEARN = 9
@@ -98,7 +99,8 @@ ARP_REPLY = '0x2'
# Map tunnel types to tables number
TUN_TABLE = {p_const.TYPE_GRE: GRE_TUN_TO_LV,
p_const.TYPE_VXLAN: VXLAN_TUN_TO_LV,
- p_const.TYPE_GENEVE: GENEVE_TUN_TO_LV}
+ p_const.TYPE_GENEVE: GENEVE_TUN_TO_LV,
+ p_const.TYPE_STT: STT_TUN_TO_LV}
# The default respawn interval for the ovsdb monitor
diff --git a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py
index 93fbc19..41d4976 100644
--- a/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py
+++ b/neutron/plugins/ml2/drivers/openvswitch/agent/ovs_neutron_agent.py
@@ -367,6 +367,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
def _reset_tunnel_ofports(self):
self.tun_br_ofports = {p_const.TYPE_GENEVE: {},
p_const.TYPE_GRE: {},
+ p_const.TYPE_STT: {},
p_const.TYPE_VXLAN: {}}
def setup_rpc(self):
@@ -629,7 +630,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
:param net_uuid: the uuid of the network associated with this vlan.
:param network_type: the network type ('gre', 'vxlan', 'vlan', 'flat',
- 'local', 'geneve')
+ 'local', 'stt', 'geneve')
:param physical_network: the physical network for 'vlan' or 'flat'
:param segmentation_id: the VID for 'vlan' or tunnel ID for 'tunnel'
'''
@@ -2110,7 +2111,7 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin,
if (self.enable_distributed_routing and self.enable_tunneling
and not self.l2_pop):
- raise ValueError(_("DVR deployments for VXLAN/GRE/Geneve "
+ raise ValueError(_("DVR deployments for VXLAN/GRE/STT/Geneve "
"underlays require L2-pop to be enabled, "
"in both the Agent and Server side."))
diff --git a/neutron/plugins/ml2/drivers/type_stt.py b/neutron/plugins/ml2/drivers/type_stt.py
new file mode 100644
index 0000000..27b8e9a
--- /dev/null
+++ b/neutron/plugins/ml2/drivers/type_stt.py
@@ -0,0 +1,142 @@
+# Copyright (c) 2015 OpenStack Foundation
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+from six import moves
+import sqlalchemy as sa
+from sqlalchemy import sql
+
+from neutron.common import exceptions as n_exc
+from neutron.db import api as db_api
+from neutron.db import model_base
+from neutron.i18n import _LE
+from neutron.plugins.common import constants as p_const
+from neutron.plugins.ml2.drivers import type_tunnel
+
+LOG = log.getLogger(__name__)
+
+stt_opts = [
+ cfg.ListOpt('context_id_ranges',
+ default=['1000:3000'],
+ help=_("Comma-separated list of "
+ "<context_id_min>:<context_id_max> tuples "
+ "enumerating ranges of STT Context IDs that are "
+ "available for tenant network allocation"))
+]
+
+cfg.CONF.register_opts(stt_opts, "ml2_type_stt")
+
+
+class SttAllocation(model_base.BASEV2):
+
+ __tablename__ = 'ml2_stt_allocations'
+
+ stt_context_id = sa.Column(sa.Integer, nullable=False, primary_key=True,
+ autoincrement=False)
+ allocated = sa.Column(sa.Boolean, nullable=False, default=False,
+ server_default=sql.false(), index=True)
+
+
+class SttEndpoints(model_base.BASEV2):
+ """Represents tunnel endpoint in RPC mode."""
+
+ __tablename__ = 'ml2_stt_endpoints'
+ __table_args__ = (
+ sa.UniqueConstraint('host',
+ name='unique_ml2_stt_endpoints0host'),
+ model_base.BASEV2.__table_args__
+ )
+ ip_address = sa.Column(sa.String(64), primary_key=True)
+ host = sa.Column(sa.String(255), nullable=True)
+
+ def __repr__(self):
+ return "<SttTunnelEndpoint(%s)>" % self.ip_address
+
+
+class SttTypeDriver(type_tunnel.EndpointTunnelTypeDriver):
+
+ def __init__(self):
+ super(SttTypeDriver, self).__init__(SttAllocation,
+ SttEndpoints)
+
+ def get_type(self):
+ return p_const.TYPE_STT
+
+ def initialize(self):
+ try:
+ self._initialize(cfg.CONF.ml2_type_stt.context_id_ranges)
+ except n_exc.NetworkTunnelRangeError:
+ LOG.exception(_LE("Failed to parse context_id_ranges. "
+ "Service terminated!"))
+ raise SystemExit()
+
+ def sync_allocations(self):
+ # FIXME(arosen) - refactor out duplicate code from type_vxlan driver.
+
+ # determine current configured allocatable context_ids
+ stt_context_ids = set()
+ for tun_min, tun_max in self.tunnel_ranges:
+ if tun_max + 1 - tun_min > p_const.MAX_STT_CONTEXT_ID:
+ LOG.error(_LE("Skipping unreasonable STT VNI range "
+ "%(tun_min)s:%(tun_max)s"),
+ {'tun_min': tun_min, 'tun_max': tun_max})
+ else:
+ stt_context_ids |= set(moves.range(tun_min, tun_max + 1))
+
+ session = db_api.get_session()
+ with session.begin(subtransactions=True):
+ # remove from table unallocated tunnels not currently allocatable
+ # fetch results as list via all() because we'll be iterating
+ # through them twice
+ allocs = (session.query(SttAllocation).
+ with_lockmode("update").all())
+ # collect all context_ids present in db
+ existing_context_ids = set(alloc.stt_context_id
+ for alloc in allocs)
+ # collect those context_ids that needs to be deleted from db
+ context_ids_to_remove = [
+ alloc.stt_context_id for alloc in allocs
+ if (alloc.stt_context_id not in stt_context_ids and
+ not alloc.allocated)]
+ # Immediately delete context_ids in chunks. This leaves no work for
+ # flush at the end of transaction
+ bulk_size = 100
+ chunked_context_ids = (
+ context_ids_to_remove[i:i + bulk_size]
+ for i in range(0, len(context_ids_to_remove), bulk_size))
+ for context_id_list in chunked_context_ids:
+ if context_id_list:
+ session.query(SttAllocation).filter(
+ SttAllocation.stt_context_id.in_(context_id_list)).delete(
+ synchronize_session=False)
+ # collect context_ids that need to be added
+ context_ids = list(stt_context_ids - existing_context_ids)
+ chunked_context_ids = (context_ids[i:i + bulk_size] for i in
+ range(0, len(context_ids), bulk_size))
+ for context_id_list in chunked_context_ids:
+ bulk = [{'stt_context_id': context_id, 'allocated': False}
+ for context_id in context_id_list]
+ session.execute(SttAllocation.__table__.insert(), bulk)
+
+ def get_endpoints(self):
+ """Get every stt endpoints from database."""
+ stt_endpoints = self._get_endpoints()
+ return [{'ip_address': stt_endpoint.ip_address,
+ 'host': stt_endpoint.host}
+ for stt_endpoint in stt_endpoints]
+
+ def add_endpoint(self, ip, host):
+ return self._add_endpoint(ip, host)
diff --git a/setup.cfg b/setup.cfg
index 5479a7c..7098b9d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -91,6 +91,7 @@ neutron.ml2.type_drivers =
geneve = neutron.plugins.ml2.drivers.type_geneve:GeneveTypeDriver
gre = neutron.plugins.ml2.drivers.type_gre:GreTypeDriver
vxlan = neutron.plugins.ml2.drivers.type_vxlan:VxlanTypeDriver
+ stt = neutron.plugins.ml2.drivers.type_stt:SttTypeDriver
neutron.ml2.mechanism_drivers =
logger = neutron.tests.unit.plugins.ml2.drivers.mechanism_logger:LoggerMechanismDriver
test = neutron.tests.unit.plugins.ml2.drivers.mechanism_test:TestMechanismDriver
测试结果
ubuntu@i1:~$ sudo iperf -c 10.0.1.30
sudo: unable to resolve host i1
------------------------------------------------------------
Client connecting to 10.0.1.30, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.32 port 41133 connected with 10.0.1.30 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 8.20 GBytes 7.04 Gbits/sec
ubuntu@i1:~$ sudo iperf -c 10.230.56.104
sudo: unable to resolve host i1
------------------------------------------------------------
Client connecting to 10.230.56.104, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.32 port 50075 connected with 10.230.56.104 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 6.83 GBytes 5.87 Gbits/sec
ubuntu@duduo:~$ sudo ip netns exec qrouter-17962b29-11de-48b4-b1e2-2015b21d5f61 iperf -c 10.0.1.30
------------------------------------------------------------
Client connecting to 10.0.1.30, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.1 port 54062 connected with 10.0.1.30 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 8.65 GBytes 7.43 Gbits/sec
ubuntu@duduo:~$ sudo iperf -c 10.230.56.104
------------------------------------------------------------
Client connecting to 10.230.56.104, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.230.56.15 port 53513 connected with 10.230.56.104 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 1.10 GBytes 943 Mbits/sec
ubuntu@i2:~$ wget http://releases.ubuntu.com/16.04/ubuntu-16.04-desktop-amd64.iso?_ga=1.132133774.10900695.1465706260
--2016-06-17 09:04:23-- http://releases.ubuntu.com/16.04/ubuntu-16.04-desktop-amd64.iso?_ga=1.132133774.10900695.1465706260
Resolving releases.ubuntu.com (releases.ubuntu.com)... 91.189.88.148, 2001:67c:1360:8c01::1d
Connecting to releases.ubuntu.com (releases.ubuntu.com)|91.189.88.148|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1485881344 (1.4G) [application/x-iso9660-image]
Saving to: 'ubuntu-16.04-desktop-amd64.iso?_ga=1.132133774.10900695.1465706260'
100%[=============================================================>] 1,485,881,344 19.3MB/s in 1m 53s
2016-06-17 09:06:17 (12.5 MB/s) - 'ubuntu-16.04-desktop-amd64.iso?_ga=1.132133774.10900695.1465706260' saved [1485881344/1485881344]
ubuntu@i1:~$ curl -O -k http://10.0.1.30:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 80.0M 0 0:00:17 0:00:17 --:--:-- 56.5M
ubuntu@i1:~$ curl -O -k http://10.230.56.104:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 83.2M 0 0:00:17 0:00:17 --:--:-- 58.0M
ubuntu@duduo:~$ sudo ip netns exec qrouter-17962b29-11de-48b4-b1e2-2015b21d5f61 curl -O -k http://10.0.1.30:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 990M 0 0:00:01 0:00:01 --:--:-- 989M
ubuntu@duduo:~$ sudo ip netns exec qrouter-17962b29-11de-48b4-b1e2-2015b21d5f61 curl -O -k http://10.230.56.104:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 664M 0 0:00:02 0:00:02 --:--:-- 664M
ubuntu@duduo:~$ sudo curl -O -k http://10.230.56.104:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 104M 0 0:00:13 0:00:13 --:--:-- 112M
ubuntu@duduo:~$ sudo ip link set mtu 9000 dev eth0
ubuntu@voltorb:~$ sudo ip link set mtu 9000 dev eth0
ubuntu@i1:~$ sudo iperf -c 10.230.56.104
sudo: unable to resolve host i1
------------------------------------------------------------
Client connecting to 10.230.56.104, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.32 port 50078 connected with 10.230.56.104 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 6.29 GBytes 5.40 Gbits/sec
ubuntu@i1:~$ sudo iperf -c 10.0.1.30
sudo: unable to resolve host i1
------------------------------------------------------------
Client connecting to 10.0.1.30, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.32 port 41136 connected with 10.0.1.30 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 5.57 GBytes 4.78 Gbits/sec
ubuntu@i1:~$ curl -O -k http://10.230.56.104:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 78.8M 0 0:00:17 0:00:17 --:--:-- 55.7M
ubuntu@i1:~$ curl -O -k http://10.0.1.30:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 79.9M 0 0:00:17 0:00:17 --:--:-- 60.3M
ubuntu@duduo:~$ sudo ethtool -K eth0 gro off
ubuntu@duduo:~$ sudo ethtool -K eth0 gso off
ubuntu@duduo:~$ sudo ethtool -K eth0 tso off
ubuntu@voltorb:~$ sudo ethtool -K eth0 gro off
ubuntu@voltorb:~$ sudo ethtool -K eth0 tso off
ubuntu@voltorb:~$ sudo ethtool -K eth0 gso off
ubuntu@i1:~$ sudo iperf -c 10.0.1.30
sudo: unable to resolve host i1
------------------------------------------------------------
Client connecting to 10.0.1.30, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.32 port 41142 connected with 10.0.1.30 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 6.88 GBytes 5.91 Gbits/sec
ubuntu@i1:~$ sudo iperf -c 10.230.56.104
sudo: unable to resolve host i1
------------------------------------------------------------
Client connecting to 10.230.56.104, TCP port 5001
TCP window size: 85.0 KByte (default)
------------------------------------------------------------
[ 3] local 10.0.1.32 port 50086 connected with 10.230.56.104 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0-10.0 sec 6.60 GBytes 5.67 Gbits/sec
ubuntu@i1:~$ curl -O -k http://10.230.56.104:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 82.0M 0 0:00:17 0:00:17 --:--:-- 60.8M
ubuntu@i1:~$ curl -O -k http://10.0.1.30:8000/ubuntu-16.04-desktop-amd64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1417M 100 1417M 0 0 83.6M 0 0:00:16 0:00:16 --:--:-- 61.4M