80 	struct sw_flow *flow;  in ovs_flow_alloc()  local
84 	flow = kmem_cache_alloc(flow_cache, GFP_KERNEL);  in ovs_flow_alloc()
85 	if (!flow)  in ovs_flow_alloc()
88 	flow->sf_acts = NULL;  in ovs_flow_alloc()
89 	flow->mask = NULL;  in ovs_flow_alloc()
90 	flow->id.unmasked_key = NULL;  in ovs_flow_alloc()
91 	flow->id.ufid_len = 0;  in ovs_flow_alloc()
92 	flow->stats_last_writer = NUMA_NO_NODE;  in ovs_flow_alloc()
103 	RCU_INIT_POINTER(flow->stats[0], stats);  in ovs_flow_alloc()
107 			RCU_INIT_POINTER(flow->stats[node], NULL);  in ovs_flow_alloc()
109 	return flow;  in ovs_flow_alloc()
111 	kmem_cache_free(flow_cache, flow);  in ovs_flow_alloc()
143 static void flow_free(struct sw_flow *flow)  in flow_free()  argument
147 	if (ovs_identifier_is_key(&flow->id))  in flow_free()
148 		kfree(flow->id.unmasked_key);  in flow_free()
149 	if (flow->sf_acts)  in flow_free()
150 		ovs_nla_free_flow_actions((struct sw_flow_actions __force *)flow->sf_acts);  in flow_free()
152 		if (flow->stats[node])  in flow_free()
154 					(struct flow_stats __force *)flow->stats[node]);  in flow_free()
155 	kmem_cache_free(flow_cache, flow);  in flow_free()
160 	struct sw_flow *flow = container_of(rcu, struct sw_flow, rcu);  in rcu_free_flow_callback()  local
162 	flow_free(flow);  in rcu_free_flow_callback()
165 void ovs_flow_free(struct sw_flow *flow, bool deferred)  in ovs_flow_free()  argument
167 	if (!flow)  in ovs_flow_free()
171 		call_rcu(&flow->rcu, rcu_free_flow_callback);  in ovs_flow_free()
173 		flow_free(flow);  in ovs_flow_free()
256 		struct sw_flow *flow;  in table_instance_destroy()  local
262 		hlist_for_each_entry_safe(flow, n, head, flow_table.node[ver]) {  in table_instance_destroy()
263 			hlist_del_rcu(&flow->flow_table.node[ver]);  in table_instance_destroy()
264 			if (ovs_identifier_is_ufid(&flow->id))  in table_instance_destroy()
265 				hlist_del_rcu(&flow->ufid_table.node[ufid_ver]);  in table_instance_destroy()
266 			ovs_flow_free(flow, deferred);  in table_instance_destroy()
294 	struct sw_flow *flow;  in ovs_flow_tbl_dump_next()  local
303 		hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) {  in ovs_flow_tbl_dump_next()
309 			return flow;  in ovs_flow_tbl_dump_next()
326 				  struct sw_flow *flow)  in table_instance_insert()  argument
330 	head = find_bucket(ti, flow->flow_table.hash);  in table_instance_insert()
331 	hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head);  in table_instance_insert()
335 				       struct sw_flow *flow)  in ufid_table_instance_insert()  argument
339 	head = find_bucket(ti, flow->ufid_table.hash);  in ufid_table_instance_insert()
340 	hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head);  in ufid_table_instance_insert()
354 		struct sw_flow *flow;  in flow_table_copy_flows()  local
360 			hlist_for_each_entry(flow, head,  in flow_table_copy_flows()
362 				ufid_table_instance_insert(new, flow);  in flow_table_copy_flows()
364 			hlist_for_each_entry(flow, head,  in flow_table_copy_flows()
366 				table_instance_insert(new, flow);  in flow_table_copy_flows()
453 static bool flow_cmp_masked_key(const struct sw_flow *flow,  in flow_cmp_masked_key()  argument
457 	return cmp_key(&flow->key, key, range->start, range->end);  in flow_cmp_masked_key()
460 static bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow,  in ovs_flow_cmp_unmasked_key()  argument
467 	BUG_ON(ovs_identifier_is_ufid(&flow->id));  in ovs_flow_cmp_unmasked_key()
468 	return cmp_key(flow->id.unmasked_key, key, key_start, key_end);  in ovs_flow_cmp_unmasked_key()
475 	struct sw_flow *flow;  in masked_flow_lookup()  local
483 	hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver]) {  in masked_flow_lookup()
484 		if (flow->mask == mask && flow->flow_table.hash == hash &&  in masked_flow_lookup()
485 		    flow_cmp_masked_key(flow, &masked_key, &mask->range))  in masked_flow_lookup()
486 			return flow;  in masked_flow_lookup()
497 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_stats()  local
502 		flow = masked_flow_lookup(ti, key, mask);  in ovs_flow_tbl_lookup_stats()
503 		if (flow)  /* Found */  in ovs_flow_tbl_lookup_stats()
504 			return flow;  in ovs_flow_tbl_lookup_stats()
522 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_exact()  local
526 		flow = masked_flow_lookup(ti, match->key, mask);  in ovs_flow_tbl_lookup_exact()
527 		if (flow && ovs_identifier_is_key(&flow->id) &&  in ovs_flow_tbl_lookup_exact()
528 		    ovs_flow_cmp_unmasked_key(flow, match))  in ovs_flow_tbl_lookup_exact()
529 			return flow;  in ovs_flow_tbl_lookup_exact()
539 static bool ovs_flow_cmp_ufid(const struct sw_flow *flow,  in ovs_flow_cmp_ufid()  argument
542 	if (flow->id.ufid_len != sfid->ufid_len)  in ovs_flow_cmp_ufid()
545 	return !memcmp(flow->id.ufid, sfid->ufid, sfid->ufid_len);  in ovs_flow_cmp_ufid()
548 bool ovs_flow_cmp(const struct sw_flow *flow, const struct sw_flow_match *match)  in ovs_flow_cmp()  argument
550 	if (ovs_identifier_is_ufid(&flow->id))  in ovs_flow_cmp()
551 		return flow_cmp_masked_key(flow, match->key, &match->range);  in ovs_flow_cmp()
553 	return ovs_flow_cmp_unmasked_key(flow, match);  in ovs_flow_cmp()
560 	struct sw_flow *flow;  in ovs_flow_tbl_lookup_ufid()  local
566 	hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver]) {  in ovs_flow_tbl_lookup_ufid()
567 		if (flow->ufid_table.hash == hash &&  in ovs_flow_tbl_lookup_ufid()
568 		    ovs_flow_cmp_ufid(flow, ufid))  in ovs_flow_tbl_lookup_ufid()
569 			return flow;  in ovs_flow_tbl_lookup_ufid()
610 void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)  in ovs_flow_tbl_remove()  argument
616 	hlist_del_rcu(&flow->flow_table.node[ti->node_ver]);  in ovs_flow_tbl_remove()
618 	if (ovs_identifier_is_ufid(&flow->id)) {  in ovs_flow_tbl_remove()
619 		hlist_del_rcu(&flow->ufid_table.node[ufid_ti->node_ver]);  in ovs_flow_tbl_remove()
626 	flow_mask_remove(table, flow->mask);  in ovs_flow_tbl_remove()
667 static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow,  in flow_mask_insert()  argument
685 	flow->mask = mask;  in flow_mask_insert()
690 static void flow_key_insert(struct flow_table *table, struct sw_flow *flow)  in flow_key_insert()  argument
695 	flow->flow_table.hash = flow_hash(&flow->key, &flow->mask->range);  in flow_key_insert()
697 	table_instance_insert(ti, flow);  in flow_key_insert()
714 static void flow_ufid_insert(struct flow_table *table, struct sw_flow *flow)  in flow_ufid_insert()  argument
718 	flow->ufid_table.hash = ufid_hash(&flow->id);  in flow_ufid_insert()
720 	ufid_table_instance_insert(ti, flow);  in flow_ufid_insert()
736 int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,  in ovs_flow_tbl_insert()  argument
741 	err = flow_mask_insert(table, flow, mask);  in ovs_flow_tbl_insert()
744 	flow_key_insert(table, flow);  in ovs_flow_tbl_insert()
745 	if (ovs_identifier_is_ufid(&flow->id))  in ovs_flow_tbl_insert()
746 		flow_ufid_insert(table, flow);  in ovs_flow_tbl_insert()