79 	struct sw_flow *flow;  in ovs_flow_alloc()  local
83 	flow = kmem_cache_alloc(flow_cache, GFP_KERNEL);  in ovs_flow_alloc()
84 	if (!flow)  in ovs_flow_alloc()
87 	flow->sf_acts = NULL;  in ovs_flow_alloc()
88 	flow->mask = NULL;  in ovs_flow_alloc()
89 	flow->id.unmasked_key = NULL;  in ovs_flow_alloc()
90 	flow->id.ufid_len = 0;  in ovs_flow_alloc()
91 	flow->stats_last_writer = NUMA_NO_NODE;  in ovs_flow_alloc()
102 	RCU_INIT_POINTER(flow->stats[0], stats);  in ovs_flow_alloc()
106 			RCU_INIT_POINTER(flow->stats[node], NULL);  in ovs_flow_alloc()
108 	return flow;  in ovs_flow_alloc()
110 	kmem_cache_free(flow_cache, flow);  in ovs_flow_alloc()
142 static void flow_free(struct sw_flow *flow)  in flow_free()  argument
146 	if (ovs_identifier_is_key(&flow->id))  in flow_free()
147 		kfree(flow->id.unmasked_key);  in flow_free()
148 	kfree((struct sw_flow_actions __force *)flow->sf_acts);  in flow_free()
150 		if (flow->stats[node])  in flow_free()
152 					(struct flow_stats __force *)flow->stats[node]);  in flow_free()
153 	kmem_cache_free(flow_cache, flow);  in flow_free()
158 	struct sw_flow *flow = container_of(rcu, struct sw_flow, rcu);  in rcu_free_flow_callback()  local
160 	flow_free(flow);  in rcu_free_flow_callback()
163 void ovs_flow_free(struct sw_flow *flow, bool deferred)  in ovs_flow_free()  argument
165 	if (!flow)  in ovs_flow_free()
169 		call_rcu(&flow->rcu, rcu_free_flow_callback);  in ovs_flow_free()
171 		flow_free(flow);  in ovs_flow_free()
254 		struct sw_flow *flow;  in table_instance_destroy()  local
260 		hlist_for_each_entry_safe(flow, n, head, flow_table.node[ver]) {  in table_instance_destroy()
261 			hlist_del_rcu(&flow->flow_table.node[ver]);  in table_instance_destroy()
262 			if (ovs_identifier_is_ufid(&flow->id))  in table_instance_destroy()
263 				hlist_del_rcu(&flow->ufid_table.node[ufid_ver]);  in table_instance_destroy()
264 			ovs_flow_free(flow, deferred);  in table_instance_destroy()
292 	struct sw_flow *flow;  in ovs_flow_tbl_dump_next()  local
301 		hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) {  in ovs_flow_tbl_dump_next()
307 			return flow;  in ovs_flow_tbl_dump_next()
324 				  struct sw_flow *flow)  in table_instance_insert()  argument
328 	head = find_bucket(ti, flow->flow_table.hash);  in table_instance_insert()
329 	hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head);  in table_instance_insert()
333 				       struct sw_flow *flow)  in ufid_table_instance_insert()  argument
337 	head = find_bucket(ti, flow->ufid_table.hash);  in ufid_table_instance_insert()
338 	hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head);  in ufid_table_instance_insert()
352 		struct sw_flow *flow;  in flow_table_copy_flows()  local
358 			hlist_for_each_entry(flow, head,  in flow_table_copy_flows()
360 				ufid_table_instance_insert(new, flow);  in flow_table_copy_flows()
362 			hlist_for_each_entry(flow, head,  in flow_table_copy_flows()
364 				table_instance_insert(new, flow);  in flow_table_copy_flows()
451 static bool flow_cmp_masked_key(const struct sw_flow *flow,  in flow_cmp_masked_key()  argument
455 	return cmp_key(&flow->key, key, range->start, range->end);  in flow_cmp_masked_key()
458 static bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow,  in ovs_flow_cmp_unmasked_key()  argument
465 	BUG_ON(ovs_identifier_is_ufid(&flow->id));  in ovs_flow_cmp_unmasked_key()
466 	return cmp_key(flow->id.unmasked_key, key, key_start, key_end);  in ovs_flow_cmp_unmasked_key()
473 	struct sw_flow *flow;  in masked_flow_lookup()  local
481 	hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver]) {  in masked_flow_lookup()
482 		if (flow->mask == mask && flow->flow_table.hash == hash &&  in masked_flow_lookup()
483 		    flow_cmp_masked_key(flow, &masked_key, &mask->range))  in masked_flow_lookup()
484 			return flow;  in masked_flow_lookup()
495 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_stats()  local
500 		flow = masked_flow_lookup(ti, key, mask);  in ovs_flow_tbl_lookup_stats()
501 		if (flow)  /* Found */  in ovs_flow_tbl_lookup_stats()
502 			return flow;  in ovs_flow_tbl_lookup_stats()
520 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_exact()  local
524 		flow = masked_flow_lookup(ti, match->key, mask);  in ovs_flow_tbl_lookup_exact()
525 		if (flow && ovs_identifier_is_key(&flow->id) &&  in ovs_flow_tbl_lookup_exact()
526 		    ovs_flow_cmp_unmasked_key(flow, match))  in ovs_flow_tbl_lookup_exact()
527 			return flow;  in ovs_flow_tbl_lookup_exact()
537 static bool ovs_flow_cmp_ufid(const struct sw_flow *flow,  in ovs_flow_cmp_ufid()  argument
540 	if (flow->id.ufid_len != sfid->ufid_len)  in ovs_flow_cmp_ufid()
543 	return !memcmp(flow->id.ufid, sfid->ufid, sfid->ufid_len);  in ovs_flow_cmp_ufid()
546 bool ovs_flow_cmp(const struct sw_flow *flow, const struct sw_flow_match *match)  in ovs_flow_cmp()  argument
548 	if (ovs_identifier_is_ufid(&flow->id))  in ovs_flow_cmp()
549 		return flow_cmp_masked_key(flow, match->key, &match->range);  in ovs_flow_cmp()
551 	return ovs_flow_cmp_unmasked_key(flow, match);  in ovs_flow_cmp()
558 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_ufid()  local
564 	hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver]) {  in ovs_flow_tbl_lookup_ufid()
565 		if (flow->ufid_table.hash == hash &&  in ovs_flow_tbl_lookup_ufid()
566 		    ovs_flow_cmp_ufid(flow, ufid))  in ovs_flow_tbl_lookup_ufid()
567 			return flow;  in ovs_flow_tbl_lookup_ufid()
608 void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)  in ovs_flow_tbl_remove()  argument
614 	hlist_del_rcu(&flow->flow_table.node[ti->node_ver]);  in ovs_flow_tbl_remove()
616 	if (ovs_identifier_is_ufid(&flow->id)) {  in ovs_flow_tbl_remove()
617 		hlist_del_rcu(&flow->ufid_table.node[ufid_ti->node_ver]);  in ovs_flow_tbl_remove()
624 	flow_mask_remove(table, flow->mask);  in ovs_flow_tbl_remove()
665 static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow,  in flow_mask_insert()  argument
683 	flow->mask = mask;  in flow_mask_insert()
688 static void flow_key_insert(struct flow_table *table, struct sw_flow *flow)  in flow_key_insert()  argument
693 	flow->flow_table.hash = flow_hash(&flow->key, &flow->mask->range);  in flow_key_insert()
695 	table_instance_insert(ti, flow);  in flow_key_insert()
712 static void flow_ufid_insert(struct flow_table *table, struct sw_flow *flow)  in flow_ufid_insert()  argument
716 	flow->ufid_table.hash = ufid_hash(&flow->id);  in flow_ufid_insert()
718 	ufid_table_instance_insert(ti, flow);  in flow_ufid_insert()
734 int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,  in ovs_flow_tbl_insert()  argument
739 	err = flow_mask_insert(table, flow, mask);  in ovs_flow_tbl_insert()
742 	flow_key_insert(table, flow);  in ovs_flow_tbl_insert()
743 	if (ovs_identifier_is_ufid(&flow->id))  in ovs_flow_tbl_insert()
744 		flow_ufid_insert(table, flow);  in ovs_flow_tbl_insert()