有如下三种方法可以删除openflow流表:
a. controller/ovs-ofctl主动发命令(OFPFC_DELETE or OFPFC_DELETE_STRICT)删除流表。OFPFC_DELETE_STRICT和OFPFC_DELETE 的区别是前者需要匹配所有的字段才能删除(包括优先级)，并且一次只能删除一条流表，后者要求指定的字段必须是流表的一个子集(不能指定优先级)，可以一次删除多个流表。
//比如添加如下两条流表
ovs-ofctl add-flow br10 "priority=80, in_port=2,ip, nw_dst=1.2.0.0/16, action=1"
ovs-ofctl add-flow br10 "priority=100, in_port=2, ip, nw_dst=1.1.1.0/24, action=1"
//使用OFPFC_DELETE删除流表时，可以只指定in_port或者ip就可以将上面两条流表删除
ovs-ofctl del-flows br10 "in_port=2"
ovs-ofctl del-flows br10 "ip"
//使用OFPFC_DELETE_STRICT(加上选项 --strict)删除流表时，需要指定流表的所有内容，包括优先级
ovs-ofctl --strict del-flows br10 "priority=100, in_port=2, ip, nw_dst=1.1.1.0/24"
b. 流表的超时机制: 添加流表时如果指定idle_timeout或者hard_timeout参数，则流表超时后将被删除。如果不指定这俩参数，则默认不会被超时机制删除。hard_timeout指定的超时时间是从创建流表，或者修改流表开始计时，超时时间到后，不管此流表有没有被使用，都会被删除。idle_timeout指定流表空闲超时时间，从最近流表被使用开始计时，如果指定时间内此流表没有被使用，则被删除。
c. 强制回收机制: 添加流表时，如果当前流表个数大于等于最大流表个数，则判断是否可以强制回收之前添加的流表。可以通过Flow_Table里的overflow_policy参数指定当前流表个数大于等于最大流表个数时的行为，如果为refuse则拒绝添加新流表，如果为evict则强制删除即将超时的流表。强制回收只考虑指定了超时时间的流表。
image.png
添加流表时，如果指定了超时时间，并且流表指定的field包含group指定的field，则会将流表添加到一个分组中。具体为根据group指定的这些field计算hash值，再根据hash值到table->eviction_groups_by_id查找struct eviction_group。然后根据超时时间计算优先级，根据优先级将此流表插入struct eviction_group
struct oftable {
...
/* Eviction groups.
*
* When a flow is added that would cause the table to have more than
* 'max_flows' flows, and 'eviction_fields' is nonnull, these groups are
* used to decide which rule to evict: the rule is chosen from the eviction
* group that contains the greatest number of rules.*/
uint32_t eviction_group_id_basis; //使用random_uint32()产生的随机数
struct hmap eviction_groups_by_id;//hash表，存放struct eviction_group，struct eviction_group中存放rule
struct heap eviction_groups_by_size;//根据struct eviction_group中rule个数将struct eviction_group排序存放
...
}
针对强制删除流表，可以做如下实验
//设置最大流表个数为3，overflow_policy为强制删除
root@ubuntu:~# ovs-vsctl -- --id=@ft create Flow_Table name=test1 flow_limit=3 overflow_policy=evict -- set Bridge br10 flow_tables=0=@ft
//查看设置
root@ubuntu:~# ovs-vsctl list Flow_table
_uuid : 44527d3b-baf0-40ae-afc9-22979bea3d68
external_ids : {}
flow_limit : 3
groups : []
name : "test1"
overflow_policy : evict
prefixes : []
//当前以及有如下三条流表
root@ubuntu:~# ovs-ofctl dump-flows br10
cookie=0x0, duration=1.861s, table=0, n_packets=0, n_bytes=0, priority=100,ip,in_port="veth_l0",nw_dst=1.1.1.0/24 actions=output:"veth_r0"
cookie=0x0, duration=1.868s, table=0, n_packets=0, n_bytes=0, priority=80,ip,in_port="veth_l0",nw_dst=1.2.0.0/16 actions=output:"veth_r0"
cookie=0x0, duration=477124.741s, table=0, n_packets=6, n_bytes=364, priority=0 actions=NORMAL
//再添加新流表时，提示流表已经满了，但是前面不是已经设置强制删除了吗，为什么不生效？
//因为当前设置的三个流表都没有指定超时时间，而强制删除不会删除永久性的流表
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=100, in_port=2, ip, nw_dst=2.1.1.0/24, action=1"
OFPT_ERROR (xid=0x4): OFPFMFC_TABLE_FULL
OFPT_FLOW_MOD (xid=0x4):
(***truncated to 64 bytes from 80***)
00000000 01 0e 00 50 00 00 00 04-00 32 20 ee 00 02 00 00 |...P.....2 .....|
00000010 00 00 00 00 00 00 00 00-00 00 00 00 00 00 08 00 |................|
00000020 00 00 00 00 00 00 00 00-02 01 01 00 00 00 00 00 |................|
00000030 00 00 00 00 00 00 00 00-00 00 00 00 00 00 00 64 |...............d|
//删除所有流表，重新添加三条流表，其中两条指定了超时时间
root@ubuntu:~# ovs-ofctl del-flows br10 ""
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=0, action=normal"
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=80, hard_timeout=1000, in_port=2,ip, nw_dst=1.2.0.0/16, action=1"
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=100, hard_timeout=1000, in_port=2, ip, nw_dst=1.1.1.0/24, action=1"
//查看当前流表
root@ubuntu:~# ovs-ofctl dump-flows br10
cookie=0x0, duration=7.037s, table=0, n_packets=0, n_bytes=0, hard_timeout=1000, priority=100,ip,in_port="veth_l0",nw_dst=1.1.1.0/24 actions=output:"veth_r0"
cookie=0x0, duration=25.689s, table=0, n_packets=0, n_bytes=0, hard_timeout=1000, priority=80,ip,in_port="veth_l0",nw_dst=1.2.0.0/16 actions=output:"veth_r0"
cookie=0x0, duration=19.506s, table=0, n_packets=0, n_bytes=0, priority=0 actions=NORMAL
root@ubuntu:~#
//再次添加第四条流表可以成功
root@ubuntu:~# ovs-ofctl add-flow br10 "priority=100, in_port=2, ip, nw_dst=2.1.1.0/24, action=1"
//查看当前流表，发现之前存在时间最长的"duration=25.689s"流表被删除了
root@ubuntu:~# ovs-ofctl dump-flows br10
cookie=0x0, duration=20.731s, table=0, n_packets=0, n_bytes=0, hard_timeout=1000, priority=100,ip,in_port="veth_l0",nw_dst=1.1.1.0/24 actions=output:"veth_r0"
cookie=0x0, duration=2.579s, table=0, n_packets=0, n_bytes=0, priority=100,ip,in_port="veth_l0",nw_dst=2.1.1.0/24 actions=output:"veth_r0"
cookie=0x0, duration=33.200s, table=0, n_packets=0, n_bytes=0, priority=0 actions=NORMAL
源码分析
下面分析下这三种情况下的源码
controller/ovs-ofctl主动删除流表
ofproto_flow_mod_start
delete_flows_start_loose(ofproto, ofm);
//先根据指定的字段ofm->criteria查找需要删除的rule，这是用的是loose方式查找
collect_rules_loose(ofproto, &ofm->criteria, rules);
delete_flows_start__(ofproto, ofm->version, rules);
delete_flow_start_strict(ofproto, ofm);
//先根据指定的字段ofm->criteria查找需要删除的rule，这是用的是strict方式查找，必须精确匹配
collect_rules_strict(ofproto, &ofm->criteria, rules);
delete_flows_start__(ofproto, ofm->version, rules);
//将rules从ofproto中删除
static void
delete_flows_start__(struct ofproto *ofproto, ovs_version_t version,
const struct rule_collection *rules)
struct rule *rule;
RULE_COLLECTION_FOR_EACH (rule, rules) {
struct oftable *table = &ofproto->tables[rule->table_id];
//流表个数减一
table->n_flows--;
//设置versions->remove_version为version，表示此流表从version开始就被删除了
cls_rule_make_invisible_in_version(&rule->cr, version);
struct cls_match *cls_match = get_cls_match_protected(rule);
cls_match_set_remove_version(cls_match, remove_version);
versions_set_remove_version(&rule->versions, version);
atomic_store_relaxed(&versions->remove_version, version);
//一个流表会插入多个地方保存，这个只是将流表从ofproto中删除，后面会将流表从分类器中删除
/* Removes 'rule' from the ofproto data structures. Caller may have deferred
* the removal from the classifier. */
/* Remove rule from ofproto data structures. */
ofproto_rule_remove__(ofproto, rule);
}
将流表从分类器中删除
ofproto_flow_mod_finish(ofproto, &ofm, req);
delete_flows_finish(ofproto, ofm, req);
delete_flows_finish__(ofproto, &ofm->old_rules, OFPRR_DELETE, req);
remove_rules_postponed(rules);
ovsrcu_postpone(remove_rules_rcu, rule_collection_detach(rules));
remove_rule_rcu__(rule);
cls_rule_visible_in_version(&rule->cr, OVS_VERSION_MAX)
//将rule从分类器删除
classifier_remove(&table->cls, &rule->cr)
超时机制
添加流表时，如果指定了超时时间，则会将流表挂到链表ofproto->expirable上，在ovs-vswitchd的主循环中，会周期性的调用ofproto-dpif.c中的run函数，此函数会遍历链表ofproto->expirable判断流表是否超时，如果超时则将流表删除。
run(struct ofproto *ofproto_)
long long now = time_msec();
ovs_mutex_lock(&ofproto_mutex);
LIST_FOR_EACH_SAFE (rule, next_rule, expirable, &ofproto->up.expirable) {
rule_expire(rule_dpif_cast(rule), now);
uint16_t hard_timeout, idle_timeout;
int reason = -1;
hard_timeout = rule->up.hard_timeout;
idle_timeout = rule->up.idle_timeout;
//now 表示当前时间，如果超过了hard_timeout或者idle_timeout，则将rule删除，
//并设置对应的reason
/* Has 'rule' expired? */
if (hard_timeout) {
long long int modified;
ovs_mutex_lock(&rule->up.mutex);
modified = rule->up.modified;
ovs_mutex_unlock(&rule->up.mutex);
if (now > modified + hard_timeout * 1000) {
reason = OFPRR_HARD_TIMEOUT;
}
}
if (reason < 0 && idle_timeout) {
long long int used;
ovs_mutex_lock(&rule->stats_mutex);
used = rule->stats.used;
ovs_mutex_unlock(&rule->stats_mutex);
if (now > used + idle_timeout * 1000) {
reason = OFPRR_IDLE_TIMEOUT;
}
}
if (reason >= 0) {
COVERAGE_INC(ofproto_dpif_expired);
ofproto_rule_expire(&rule->up, reason);
struct rule_collection rules;
rule_collection_init(&rules);
rule_collection_add(&rules, rule);
delete_flows__(&rules, reason, NULL);
//收集完需要删除的rules后，后面的流表和主动删除流表时一样的
if (rule_collection_n(rules)) {
struct ofproto *ofproto = rule_collection_rules(rules)[0]->ofproto;
delete_flows_start__(ofproto, ofproto->tables_version + 1, rules);
ofproto_bump_tables_version(ofproto);
delete_flows_finish__(ofproto, rules, reason, req);
ofmonitor_flush(ofproto->connmgr);
}
}
}
ovs_mutex_unlock(&ofproto_mutex);
evict强制删除
添加流表时会将流表插入eviction_group中
ofproto_rule_insert__ -> eviction_group_add_rule(rule)
/* Adds 'rule' to an appropriate eviction group for its oftable's
* configuration. Does nothing if 'rule''s oftable doesn't have eviction
* enabled, or if 'rule' is a permanent rule (one that will never expire on its
* own).
*
* The caller must ensure that 'rule' is not already in an eviction group. */
static void
eviction_group_add_rule(struct rule *rule)
OVS_REQUIRES(ofproto_mutex)
{
struct ofproto *ofproto = rule->ofproto;
struct oftable *table = &ofproto->tables[rule->table_id];
bool has_timeout;
/* Timeouts may be modified only when holding 'ofproto_mutex'. We have it
* so no additional protection is needed. */
has_timeout = rule->hard_timeout || rule->idle_timeout;
//配置了overflow_policy为evict，并且流表指定了超时时间
if (table->eviction && has_timeout) {
struct eviction_group *evg;
//使用函数 eviction_group_hash_rule 计算hash值，查找 struct eviction_group
evg = eviction_group_find(table, eviction_group_hash_rule(rule));
//使用函数 rule_eviction_priority 计算优先级，将 rule 按优先级插入 evg 中
rule->eviction_group = evg;
heap_insert(&evg->rules, &rule->evg_node,
rule_eviction_priority(ofproto, rule));
//按照 evg 中最新rule个数重新排序 table->eviction_groups_by_size
eviction_group_resized(table, evg);
}
}
//根据group配置的field计算hash值
/* Hashes the 'rule''s values for the eviction_fields of 'rule''s table, and
* returns the hash value. */
static uint32_t
eviction_group_hash_rule(struct rule *rule)
OVS_REQUIRES(ofproto_mutex)
{
struct oftable *table = &rule->ofproto->tables[rule->table_id];
const struct mf_subfield *sf;
struct flow flow;
uint32_t hash;
hash = table->eviction_group_id_basis;
miniflow_expand(rule->cr.match.flow, &flow);
for (sf = table->eviction_fields;
sf < &table->eviction_fields[table->n_eviction_fields];
sf++)
{
if (mf_are_prereqs_ok(sf->field, &flow, NULL)) {
union mf_value value;
mf_get_value(sf->field, &flow, &value);
if (sf->ofs) {
bitwise_zero(&value, sf->field->n_bytes, 0, sf->ofs);
}
if (sf->ofs + sf->n_bits < sf->field->n_bytes * 8) {
unsigned int start = sf->ofs + sf->n_bits;
bitwise_zero(&value, sf->field->n_bytes, start,
sf->field->n_bytes * 8 - start);
}
hash = hash_bytes(&value, sf->field->n_bytes, hash);
} else {
hash = hash_int(hash, 0);
}
}
return hash;
}
//根据hash值查找table->eviction_groups_by_id，如果不存在则创建新的struct eviction_group
/* Returns an eviction group within 'table' with the given 'id', creating one
* if necessary. */
static struct eviction_group *
eviction_group_find(struct oftable *table, uint32_t id)
OVS_REQUIRES(ofproto_mutex)
{
struct eviction_group *evg;
HMAP_FOR_EACH_WITH_HASH (evg, id_node, id, &table->eviction_groups_by_id) {
return evg;
}
evg = xmalloc(sizeof *evg);
hmap_insert(&table->eviction_groups_by_id, &evg->id_node, id);
heap_insert(&table->eviction_groups_by_size, &evg->size_node,
eviction_group_priority(0));
heap_init(&evg->rules);
return evg;
}
//根据rule的超时时间计算优先级
/* Returns an eviction priority for 'rule'. The return value should be
* interpreted so that higher priorities make a rule a more attractive
* candidate for eviction. */
static uint64_t
rule_eviction_priority(struct ofproto *ofproto, struct rule *rule)
OVS_REQUIRES(ofproto_mutex)
{
/* Calculate absolute time when this flow will expire. If it will never
* expire, then return 0 to make it unevictable. */
long long int expiration = LLONG_MAX;
if (rule->hard_timeout) {
/* 'modified' needs protection even when we hold 'ofproto_mutex'. */
ovs_mutex_lock(&rule->mutex);
long long int modified = rule->modified;
ovs_mutex_unlock(&rule->mutex);
expiration = modified + rule->hard_timeout * 1000;
}
if (rule->idle_timeout) {
uint64_t packets, bytes;
long long int used;
long long int idle_expiration;
ofproto->ofproto_class->rule_get_stats(rule, &packets, &bytes, &used);
idle_expiration = used + rule->idle_timeout * 1000;
expiration = MIN(expiration, idle_expiration);
}
if (expiration == LLONG_MAX) {
return 0;
}
/* Calculate the time of expiration as a number of (approximate) seconds
* after program startup.
*
* This should work OK for program runs that last UINT32_MAX seconds or
* less. Therefore, please restart OVS at least once every 136 years. */
uint32_t expiration_ofs = (expiration >> 10) - (time_boot_msec() >> 10);
/* Combine expiration time with OpenFlow "importance" to form a single
* priority value. We want flows with relatively low "importance" to be
* evicted before even considering expiration time, so put "importance" in
* the most significant bits and expiration time in the least significant
* bits.
*
* Small 'priority' should be evicted before those with large 'priority'.
* The caller expects the opposite convention (a large return value being
* more attractive for eviction) so we invert it before returning. */
uint64_t priority = ((uint64_t) rule->importance << 32) + expiration_ofs;
return UINT64_MAX - priority;
}
当再次添加流表时，如果当前流表个数超过最大流表个数，则
add_flow_start
/* If necessary, evict an existing rule to clear out space. */
if (table->n_flows >= table->max_flows) {
//选择一个要强制删除的rule，如果不能强制删除，则添加流表失败并返回OFPERR_OFPFMFC_TABLE_FULL
if (!choose_rule_to_evict(table, &old_rule)) {
return OFPERR_OFPFMFC_TABLE_FULL;
}
//将rule从 evict_group 中删除
eviction_group_remove_rule(old_rule);
/* Marks 'old_rule' as an evicted rule rather than replaced rule.
*/
old_rule->removed_reason = OFPRR_EVICTION;
}
replace_rule_start(ofproto, ofm, old_rule, new_rule);
//设置versions->remove_version为version，表示此流表从version开始就被删除了
/* Mark the old rule for removal in the next version. */
cls_rule_make_invisible_in_version(&old_rule->cr, ofm->version);
//一个流表会插入多个地方保存，这个只是将流表从ofproto中删除，后面会将流表从分类器中删除
/* Remove the old rule from data structures. */
ofproto_rule_remove__(ofproto, old_rule);
//最后再 ofproto_flow_mod_finish 中将old_rule从分类器中删除
ofproto_flow_mod_finish
/* Table overflow policy. */
/* Chooses and updates 'rulep' with a rule to evict from 'table'. Sets 'rulep'
* to NULL if the table is not configured to evict rules or if the table
* contains no evictable rules. (Rules with a readlock on their evict rwlock,
* or with no timeouts are not evictable.) */
static bool
choose_rule_to_evict(struct oftable *table, struct rule **rulep)
OVS_REQUIRES(ofproto_mutex)
{
struct eviction_group *evg;
*rulep = NULL;
//overflow_policy不为evict，则返回false，说明不能强制删除流表
if (!table->eviction) {
return false;
}
//下面的两层循环用来找到最适合被删除的rule:
//a. 从rule个数最多的group中选择，由于table->eviction_groups_by_size已经按size排序了，直接取出来即可
//b. 从 evict_group 取出优先级最高的rule，由于evg->rules也是排好序的，取第一个即可。
/* In the common case, the outer and inner loops here will each be entered
* exactly once:
*
* - The inner loop normally "return"s in its first iteration. If the
* eviction group has any evictable rules, then it always returns in
* some iteration.
*
* - The outer loop only iterates more than once if the largest eviction
* group has no evictable rules.
*
* - The outer loop can exit only if table's 'max_flows' is all filled up
* by unevictable rules. */
//外层循环取出 rule 个数最多的 group
HEAP_FOR_EACH (evg, size_node, &table->eviction_groups_by_size) {
struct rule *rule;
//取出group中，优先级最大的rule。优先级大意味着超时时间将近。
HEAP_FOR_EACH (rule, evg_node, &evg->rules) {
*rulep = rule;
return true;
}
}
return false;
}