138 static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
141 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
153 			return vport->dp;  in get_dp_rcu()
164 	struct datapath *dp;  in get_dp()  local
168 	dp = get_dp_rcu(net, dp_ifindex);  in get_dp()
171 	return dp;  in get_dp()
175 const char *ovs_dp_name(const struct datapath *dp)  in ovs_dp_name()  argument
177 	struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);  in ovs_dp_name()
181 static int get_dpifindex(const struct datapath *dp)  in get_dpifindex()  argument
188 	local = ovs_vport_rcu(dp, OVSP_LOCAL);  in get_dpifindex()
201 	struct datapath *dp = container_of(rcu, struct datapath, rcu);  in destroy_dp_rcu()  local
203 	ovs_flow_tbl_destroy(&dp->table);  in destroy_dp_rcu()
204 	free_percpu(dp->stats_percpu);  in destroy_dp_rcu()
205 	kfree(dp->ports);  in destroy_dp_rcu()
206 	kfree(dp);  in destroy_dp_rcu()
209 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,  in vport_hash_bucket()  argument
212 	return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];  in vport_hash_bucket()
216 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)  in ovs_lookup_vport()  argument
221 	head = vport_hash_bucket(dp, port_no);  in ovs_lookup_vport()
236 		struct datapath *dp = parms->dp;  in new_vport()  local
237 		struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);  in new_vport()
259 	struct datapath *dp = p->dp;  in ovs_dp_process_packet()  local
266 	stats = this_cpu_ptr(dp->stats_percpu);  in ovs_dp_process_packet()
269 	flow = ovs_flow_tbl_lookup_stats(&dp->table, key, &n_mask_hit);  in ovs_dp_process_packet()
278 		error = ovs_dp_upcall(dp, skb, key, &upcall);  in ovs_dp_process_packet()
289 	ovs_execute_actions(dp, skb, sf_acts, key);  in ovs_dp_process_packet()
301 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,  in ovs_dp_upcall()  argument
314 		err = queue_userspace_packet(dp, skb, key, upcall_info);  in ovs_dp_upcall()
316 		err = queue_gso_packets(dp, skb, key, upcall_info);  in ovs_dp_upcall()
323 	stats = this_cpu_ptr(dp->stats_percpu);  in ovs_dp_upcall()
332 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,  in queue_gso_packets()  argument
363 		err = queue_userspace_packet(dp, skb, key, upcall_info);  in queue_gso_packets()
407 static void pad_packet(struct datapath *dp, struct sk_buff *skb)  in pad_packet()  argument
409 	if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {  in pad_packet()
417 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,  in queue_userspace_packet()  argument
426 		.dst_sk = ovs_dp_get_net(dp)->genl_sock,  in queue_userspace_packet()
433 	dp_ifindex = get_dpifindex(dp);  in queue_userspace_packet()
463 	if (dp->user_features & OVS_DP_F_UNALIGNED)  in queue_userspace_packet()
513 		pad_packet(dp, user_skb);  in queue_userspace_packet()
529 	pad_packet(dp, user_skb);  in queue_userspace_packet()
533 	err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);  in queue_userspace_packet()
552 	struct datapath *dp;  in ovs_packet_cmd_execute()  local
613 	dp = get_dp_rcu(net, ovs_header->dp_ifindex);  in ovs_packet_cmd_execute()
615 	if (!dp)  in ovs_packet_cmd_execute()
618 	input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);  in ovs_packet_cmd_execute()
620 		input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);  in ovs_packet_cmd_execute()
630 	err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);  in ovs_packet_cmd_execute()
675 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,  in get_dp_stats()  argument
682 	stats->n_flows = ovs_flow_tbl_count(&dp->table);  in get_dp_stats()
683 	mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);  in get_dp_stats()
692 		percpu_stats = per_cpu_ptr(dp->stats_percpu, i);  in get_dp_stats()
915 	struct datapath *dp;  in ovs_flow_cmd_new()  local
974 	dp = get_dp(net, ovs_header->dp_ifindex);  in ovs_flow_cmd_new()
975 	if (unlikely(!dp)) {  in ovs_flow_cmd_new()
982 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);  in ovs_flow_cmd_new()
984 		flow = ovs_flow_tbl_lookup(&dp->table, &key);  in ovs_flow_cmd_new()
989 		error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);  in ovs_flow_cmd_new()
1024 				flow = ovs_flow_tbl_lookup_exact(&dp->table,  in ovs_flow_cmd_new()
1098 	struct datapath *dp;  in ovs_flow_cmd_set()  local
1140 	dp = get_dp(net, ovs_header->dp_ifindex);  in ovs_flow_cmd_set()
1141 	if (unlikely(!dp)) {  in ovs_flow_cmd_set()
1147 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);  in ovs_flow_cmd_set()
1149 		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);  in ovs_flow_cmd_set()
1210 	struct datapath *dp;  in ovs_flow_cmd_get()  local
1232 	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);  in ovs_flow_cmd_get()
1233 	if (!dp) {  in ovs_flow_cmd_get()
1239 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);  in ovs_flow_cmd_get()
1241 		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);  in ovs_flow_cmd_get()
1269 	struct datapath *dp;  in ovs_flow_cmd_del()  local
1287 	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);  in ovs_flow_cmd_del()
1288 	if (unlikely(!dp)) {  in ovs_flow_cmd_del()
1294 		err = ovs_flow_tbl_flush(&dp->table);  in ovs_flow_cmd_del()
1299 		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);  in ovs_flow_cmd_del()
1301 		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);  in ovs_flow_cmd_del()
1307 	ovs_flow_tbl_remove(&dp->table, flow);  in ovs_flow_cmd_del()
1341 	struct datapath *dp;  in ovs_flow_cmd_dump()  local
1352 	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);  in ovs_flow_cmd_dump()
1353 	if (!dp) {  in ovs_flow_cmd_dump()
1358 	ti = rcu_dereference(dp->table.ti);  in ovs_flow_cmd_dump()
1443 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,  in ovs_dp_cmd_fill_info()  argument
1456 	ovs_header->dp_ifindex = get_dpifindex(dp);  in ovs_dp_cmd_fill_info()
1458 	err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));  in ovs_dp_cmd_fill_info()
1462 	get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);  in ovs_dp_cmd_fill_info()
1472 	if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))  in ovs_dp_cmd_fill_info()
1494 	struct datapath *dp;  in lookup_datapath()  local
1497 		dp = get_dp(net, ovs_header->dp_ifindex);  in lookup_datapath()
1502 		dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;  in lookup_datapath()
1504 	return dp ? dp : ERR_PTR(-ENODEV);  in lookup_datapath()
1509 	struct datapath *dp;  in ovs_dp_reset_user_features()  local
1511 	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);  in ovs_dp_reset_user_features()
1512 	if (IS_ERR(dp))  in ovs_dp_reset_user_features()
1515 	WARN(dp->user_features, "Dropping previously announced user features\n");  in ovs_dp_reset_user_features()
1516 	dp->user_features = 0;  in ovs_dp_reset_user_features()
1519 static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])  in ovs_dp_change()  argument
1522 		dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);  in ovs_dp_change()
1530 	struct datapath *dp;  in ovs_dp_cmd_new()  local
1544 	dp = kzalloc(sizeof(*dp), GFP_KERNEL);  in ovs_dp_cmd_new()
1545 	if (dp == NULL)  in ovs_dp_cmd_new()
1548 	ovs_dp_set_net(dp, sock_net(skb->sk));  in ovs_dp_cmd_new()
1551 	err = ovs_flow_tbl_init(&dp->table);  in ovs_dp_cmd_new()
1555 	dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);  in ovs_dp_cmd_new()
1556 	if (!dp->stats_percpu) {  in ovs_dp_cmd_new()
1561 	dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),  in ovs_dp_cmd_new()
1563 	if (!dp->ports) {  in ovs_dp_cmd_new()
1569 		INIT_HLIST_HEAD(&dp->ports[i]);  in ovs_dp_cmd_new()
1575 	parms.dp = dp;  in ovs_dp_cmd_new()
1579 	ovs_dp_change(dp, a);  in ovs_dp_cmd_new()
1602 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,  in ovs_dp_cmd_new()
1606 	ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);  in ovs_dp_cmd_new()
1607 	list_add_tail_rcu(&dp->list_node, &ovs_net->dps);  in ovs_dp_cmd_new()
1616 	kfree(dp->ports);  in ovs_dp_cmd_new()
1618 	free_percpu(dp->stats_percpu);  in ovs_dp_cmd_new()
1620 	ovs_flow_tbl_destroy(&dp->table);  in ovs_dp_cmd_new()
1622 	kfree(dp);  in ovs_dp_cmd_new()
1630 static void __dp_destroy(struct datapath *dp)  in __dp_destroy()  argument
1638 		hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)  in __dp_destroy()
1643 	list_del_rcu(&dp->list_node);  in __dp_destroy()
1648 	ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));  in __dp_destroy()
1651 	call_rcu(&dp->rcu, destroy_dp_rcu);  in __dp_destroy()
1657 	struct datapath *dp;  in ovs_dp_cmd_del()  local
1665 	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);  in ovs_dp_cmd_del()
1666 	err = PTR_ERR(dp);  in ovs_dp_cmd_del()
1667 	if (IS_ERR(dp))  in ovs_dp_cmd_del()
1670 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,  in ovs_dp_cmd_del()
1674 	__dp_destroy(dp);  in ovs_dp_cmd_del()
1690 	struct datapath *dp;  in ovs_dp_cmd_set()  local
1698 	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);  in ovs_dp_cmd_set()
1699 	err = PTR_ERR(dp);  in ovs_dp_cmd_set()
1700 	if (IS_ERR(dp))  in ovs_dp_cmd_set()
1703 	ovs_dp_change(dp, info->attrs);  in ovs_dp_cmd_set()
1705 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,  in ovs_dp_cmd_set()
1723 	struct datapath *dp;  in ovs_dp_cmd_get()  local
1731 	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);  in ovs_dp_cmd_get()
1732 	if (IS_ERR(dp)) {  in ovs_dp_cmd_get()
1733 		err = PTR_ERR(dp);  in ovs_dp_cmd_get()
1736 	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,  in ovs_dp_cmd_get()
1752 	struct datapath *dp;  in ovs_dp_cmd_dump()  local
1757 	list_for_each_entry(dp, &ovs_net->dps, list_node) {  in ovs_dp_cmd_dump()
1759 		    ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,  in ovs_dp_cmd_dump()
1829 	ovs_header->dp_ifindex = get_dpifindex(vport->dp);  in ovs_vport_cmd_fill_info()
1886 	struct datapath *dp;  in lookup_vport()  local
1894 		    ovs_header->dp_ifindex != get_dpifindex(vport->dp))  in lookup_vport()
1903 		dp = get_dp(net, ovs_header->dp_ifindex);  in lookup_vport()
1904 		if (!dp)  in lookup_vport()
1907 		vport = ovs_vport_ovsl_rcu(dp, port_no);  in lookup_vport()
1922 	struct datapath *dp;  in ovs_vport_cmd_new()  local
1941 	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);  in ovs_vport_cmd_new()
1943 	if (!dp)  in ovs_vport_cmd_new()
1947 		vport = ovs_vport_ovsl(dp, port_no);  in ovs_vport_cmd_new()
1957 			vport = ovs_vport_ovsl(dp, port_no);  in ovs_vport_cmd_new()
1966 	parms.dp = dp;  in ovs_vport_cmd_new()
2114 	struct datapath *dp;  in ovs_vport_cmd_dump()  local
2119 	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);  in ovs_vport_cmd_dump()
2120 	if (!dp) {  in ovs_vport_cmd_dump()
2128 		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {  in ovs_vport_cmd_dump()
2245 	struct datapath *dp;  in list_vports_from_net()  local
2247 	list_for_each_entry(dp, &ovs_net->dps, list_node) {  in list_vports_from_net()
2253 			hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {  in list_vports_from_net()
2266 	struct datapath *dp, *dp_next;  in ovs_exit_net()  local
2274 	list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)  in ovs_exit_net()
2275 		__dp_destroy(dp);  in ovs_exit_net()