80 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init);
130 	struct mlx4_ib_dev *ibdev = to_mdev(device);  in mlx4_ib_get_netdev()  local
134 	dev = mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port_num);  in mlx4_ib_get_netdev()
137 		if (mlx4_is_bonded(ibdev->dev)) {  in mlx4_ib_get_netdev()
158 			       struct mlx4_ib_dev *ibdev,  in mlx4_ib_update_gids()  argument
163 	struct mlx4_dev *dev = ibdev->dev;  in mlx4_ib_update_gids()
197 	struct mlx4_ib_dev *ibdev = to_mdev(device);  in mlx4_ib_add_gid()  local
198 	struct mlx4_ib_iboe *iboe = &ibdev->iboe;  in mlx4_ib_add_gid()
258 		ret = mlx4_ib_update_gids(gids, ibdev, port_num);  in mlx4_ib_add_gid()
271 	struct mlx4_ib_dev *ibdev = to_mdev(device);  in mlx4_ib_del_gid()  local
272 	struct mlx4_ib_iboe *iboe = &ibdev->iboe;  in mlx4_ib_del_gid()
311 		ret = mlx4_ib_update_gids(gids, ibdev, port_num);  in mlx4_ib_del_gid()
317 int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev *ibdev,  in mlx4_ib_gid_index_to_real_index()  argument
320 	struct mlx4_ib_iboe *iboe = &ibdev->iboe;  in mlx4_ib_gid_index_to_real_index()
332 	if (mlx4_is_bonded(ibdev->dev))  in mlx4_ib_gid_index_to_real_index()
335 	if (!rdma_cap_roce_gid_table(&ibdev->ib_dev, port_num))  in mlx4_ib_gid_index_to_real_index()
338 	ret = ib_get_cached_gid(&ibdev->ib_dev, port_num, index, &gid, NULL);  in mlx4_ib_gid_index_to_real_index()
359 static int mlx4_ib_query_device(struct ib_device *ibdev,  in mlx4_ib_query_device()  argument
363 	struct mlx4_ib_dev *dev = to_mdev(ibdev);  in mlx4_ib_query_device()
397 	err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,  in mlx4_ib_query_device()
516 static int ib_link_query_port(struct ib_device *ibdev, u8 port,  in ib_link_query_port()  argument
534 	if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)  in ib_link_query_port()
537 	err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,  in ib_link_query_port()
553 		props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port];  in ib_link_query_port()
554 	props->max_msg_sz	= to_mdev(ibdev)->dev->caps.max_msg_sz;  in ib_link_query_port()
555 	props->pkey_tbl_len	= to_mdev(ibdev)->dev->caps.pkey_table_len[port];  in ib_link_query_port()
586 		err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port,  in ib_link_query_port()
611 static int eth_link_query_port(struct ib_device *ibdev, u8 port,  in eth_link_query_port()  argument
615 	struct mlx4_ib_dev *mdev = to_mdev(ibdev);  in eth_link_query_port()
668 int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,  in __mlx4_ib_query_port()  argument
675 	err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ?  in __mlx4_ib_query_port()
676 		ib_link_query_port(ibdev, port, props, netw_view) :  in __mlx4_ib_query_port()
677 				eth_link_query_port(ibdev, port, props, netw_view);  in __mlx4_ib_query_port()
682 static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port,  in mlx4_ib_query_port()  argument
686 	return __mlx4_ib_query_port(ibdev, port, props, 0);  in mlx4_ib_query_port()
689 int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,  in __mlx4_ib_query_gid()  argument
695 	struct mlx4_ib_dev *dev = to_mdev(ibdev);  in __mlx4_ib_query_gid()
745 static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,  in mlx4_ib_query_gid()  argument
750 	if (rdma_protocol_ib(ibdev, port))  in mlx4_ib_query_gid()
751 		return __mlx4_ib_query_gid(ibdev, port, index, gid, 0);  in mlx4_ib_query_gid()
753 	if (!rdma_protocol_roce(ibdev, port))  in mlx4_ib_query_gid()
756 	if (!rdma_cap_roce_gid_table(ibdev, port))  in mlx4_ib_query_gid()
759 	ret = ib_get_cached_gid(ibdev, port, index, gid, NULL);  in mlx4_ib_query_gid()
768 int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,  in __mlx4_ib_query_pkey()  argument
785 	if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)  in __mlx4_ib_query_pkey()
788 	err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,  in __mlx4_ib_query_pkey()
801 static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)  in mlx4_ib_query_pkey()  argument
803 	return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0);  in mlx4_ib_query_pkey()
806 static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,  in mlx4_ib_modify_device()  argument
818 	if (mlx4_is_slave(to_mdev(ibdev)->dev))  in mlx4_ib_modify_device()
821 	spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags);  in mlx4_ib_modify_device()
822 	memcpy(ibdev->node_desc, props->node_desc, 64);  in mlx4_ib_modify_device()
823 	spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags);  in mlx4_ib_modify_device()
829 	mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev);  in mlx4_ib_modify_device()
834 	mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0,  in mlx4_ib_modify_device()
837 	mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox);  in mlx4_ib_modify_device()
868 static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,  in mlx4_ib_modify_port()  argument
871 	struct mlx4_ib_dev *mdev = to_mdev(ibdev);  in mlx4_ib_modify_port()
886 	err = mlx4_ib_query_port(ibdev, port, &attr);  in mlx4_ib_modify_port()
898 	mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);  in mlx4_ib_modify_port()
902 static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,  in mlx4_ib_alloc_ucontext()  argument
905 	struct mlx4_ib_dev *dev = to_mdev(ibdev);  in mlx4_ib_alloc_ucontext()
914 	if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {  in mlx4_ib_alloc_ucontext()
930 	err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);  in mlx4_ib_alloc_ucontext()
939 	if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)  in mlx4_ib_alloc_ucontext()
945 		mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);  in mlx4_ib_alloc_ucontext()
1141 static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev,  in mlx4_ib_alloc_pd()  argument
1152 	err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);  in mlx4_ib_alloc_pd()
1160 			mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);  in mlx4_ib_alloc_pd()
1176 static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,  in mlx4_ib_alloc_xrcd()  argument
1184 	if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))  in mlx4_ib_alloc_xrcd()
1191 	err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);  in mlx4_ib_alloc_xrcd()
1195 	xrcd->pd = ib_alloc_pd(ibdev);  in mlx4_ib_alloc_xrcd()
1202 	xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr);  in mlx4_ib_alloc_xrcd()
1213 	mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);  in mlx4_ib_alloc_xrcd()
1252 static void mlx4_ib_delete_counters_table(struct mlx4_ib_dev *ibdev,  in mlx4_ib_delete_counters_table()  argument
1261 			mlx4_counter_free(ibdev->dev, counter->index);  in mlx4_ib_delete_counters_table()
1932 static void mlx4_ib_update_qps(struct mlx4_ib_dev *ibdev,  in mlx4_ib_update_qps()  argument
1944 	atomic64_set(&ibdev->iboe.mac[port - 1], new_smac);  in mlx4_ib_update_qps()
1947 	if (!mlx4_is_mfunc(ibdev->dev))  in mlx4_ib_update_qps()
1950 	mutex_lock(&ibdev->qp1_proxy_lock[port - 1]);  in mlx4_ib_update_qps()
1951 	qp = ibdev->qp1_proxy[port - 1];  in mlx4_ib_update_qps()
1962 		new_smac_index = mlx4_register_mac(ibdev->dev, port, new_smac);  in mlx4_ib_update_qps()
1968 		if (mlx4_update_qp(ibdev->dev, qp->mqp.qpn, MLX4_UPDATE_QP_SMAC,  in mlx4_ib_update_qps()
1983 		mlx4_unregister_mac(ibdev->dev, port, release_mac);  in mlx4_ib_update_qps()
1986 	mutex_unlock(&ibdev->qp1_proxy_lock[port - 1]);  in mlx4_ib_update_qps()
1989 static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev,  in mlx4_ib_scan_netdevs()  argument
2000 	iboe = &ibdev->iboe;  in mlx4_ib_scan_netdevs()
2003 	mlx4_foreach_ib_transport_port(port, ibdev->dev) {  in mlx4_ib_scan_netdevs()
2006 			mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port);  in mlx4_ib_scan_netdevs()
2017 		mlx4_ib_update_qps(ibdev, dev, update_qps_port);  in mlx4_ib_scan_netdevs()
2024 	struct mlx4_ib_dev *ibdev;  in mlx4_ib_netdev_event()  local
2029 	ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb);  in mlx4_ib_netdev_event()
2030 	mlx4_ib_scan_netdevs(ibdev, dev, event);  in mlx4_ib_netdev_event()
2035 static void init_pkeys(struct mlx4_ib_dev *ibdev)  in init_pkeys()  argument
2041 	if (mlx4_is_master(ibdev->dev)) {  in init_pkeys()
2042 		for (slave = 0; slave <= ibdev->dev->persist->num_vfs;  in init_pkeys()
2044 			for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {  in init_pkeys()
2046 				     i < ibdev->dev->phys_caps.pkey_phys_table_len[port];  in init_pkeys()
2048 					ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] =  in init_pkeys()
2050 						(slave == mlx4_master_func_num(ibdev->dev) || !i) ? i :  in init_pkeys()
2051 							ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1;  in init_pkeys()
2052 					mlx4_sync_pkey_table(ibdev->dev, slave, port, i,  in init_pkeys()
2053 							     ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]);  in init_pkeys()
2058 		for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {  in init_pkeys()
2060 			     i < ibdev->dev->phys_caps.pkey_phys_table_len[port];  in init_pkeys()
2062 				ibdev->pkeys.phys_pkey_cache[port-1][i] =  in init_pkeys()
2068 static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)  in mlx4_ib_alloc_eqs()  argument
2072 	ibdev->eq_table = kcalloc(dev->caps.num_comp_vectors,  in mlx4_ib_alloc_eqs()
2073 				  sizeof(ibdev->eq_table[0]), GFP_KERNEL);  in mlx4_ib_alloc_eqs()
2074 	if (!ibdev->eq_table)  in mlx4_ib_alloc_eqs()
2082 			ibdev->eq_table[eq] = total_eqs;  in mlx4_ib_alloc_eqs()
2084 					    &ibdev->eq_table[eq]))  in mlx4_ib_alloc_eqs()
2087 				ibdev->eq_table[eq] = -1;  in mlx4_ib_alloc_eqs()
2092 	     ibdev->eq_table[i++] = -1)  in mlx4_ib_alloc_eqs()
2096 	ibdev->ib_dev.num_comp_vectors = eq;  in mlx4_ib_alloc_eqs()
2099 static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)  in mlx4_ib_free_eqs()  argument
2102 	int total_eqs = ibdev->ib_dev.num_comp_vectors;  in mlx4_ib_free_eqs()
2105 	if (!ibdev->eq_table)  in mlx4_ib_free_eqs()
2109 	ibdev->ib_dev.num_comp_vectors = 0;  in mlx4_ib_free_eqs()
2112 		mlx4_release_eq(dev, ibdev->eq_table[i]);  in mlx4_ib_free_eqs()
2114 	kfree(ibdev->eq_table);  in mlx4_ib_free_eqs()
2115 	ibdev->eq_table = NULL;  in mlx4_ib_free_eqs()
2118 static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num,  in mlx4_port_immutable()  argument
2124 	err = mlx4_ib_query_port(ibdev, port_num, &attr);  in mlx4_port_immutable()
2131 	if (mlx4_ib_port_link_layer(ibdev, port_num) == IB_LINK_LAYER_INFINIBAND)  in mlx4_port_immutable()
2143 	struct mlx4_ib_dev *ibdev;  in mlx4_ib_add()  local
2164 	ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev);  in mlx4_ib_add()
2165 	if (!ibdev) {  in mlx4_ib_add()
2171 	iboe = &ibdev->iboe;  in mlx4_ib_add()
2173 	if (mlx4_pd_alloc(dev, &ibdev->priv_pdn))  in mlx4_ib_add()
2176 	if (mlx4_uar_alloc(dev, &ibdev->priv_uar))  in mlx4_ib_add()
2179 	ibdev->uar_map = ioremap((phys_addr_t) ibdev->priv_uar.pfn << PAGE_SHIFT,  in mlx4_ib_add()
2181 	if (!ibdev->uar_map)  in mlx4_ib_add()
2183 	MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);  in mlx4_ib_add()
2185 	ibdev->dev = dev;  in mlx4_ib_add()
2186 	ibdev->bond_next_port	= 0;  in mlx4_ib_add()
2188 	strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);  in mlx4_ib_add()
2189 	ibdev->ib_dev.owner		= THIS_MODULE;  in mlx4_ib_add()
2190 	ibdev->ib_dev.node_type		= RDMA_NODE_IB_CA;  in mlx4_ib_add()
2191 	ibdev->ib_dev.local_dma_lkey	= dev->caps.reserved_lkey;  in mlx4_ib_add()
2192 	ibdev->num_ports		= num_ports;  in mlx4_ib_add()
2193 	ibdev->ib_dev.phys_port_cnt     = mlx4_is_bonded(dev) ?  in mlx4_ib_add()
2194 						1 : ibdev->num_ports;  in mlx4_ib_add()
2195 	ibdev->ib_dev.num_comp_vectors	= dev->caps.num_comp_vectors;  in mlx4_ib_add()
2196 	ibdev->ib_dev.dma_device	= &dev->persist->pdev->dev;  in mlx4_ib_add()
2197 	ibdev->ib_dev.get_netdev	= mlx4_ib_get_netdev;  in mlx4_ib_add()
2198 	ibdev->ib_dev.add_gid		= mlx4_ib_add_gid;  in mlx4_ib_add()
2199 	ibdev->ib_dev.del_gid		= mlx4_ib_del_gid;  in mlx4_ib_add()
2202 		ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION;  in mlx4_ib_add()
2204 		ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;  in mlx4_ib_add()
2206 	ibdev->ib_dev.uverbs_cmd_mask	=  in mlx4_ib_add()
2232 	ibdev->ib_dev.query_device	= mlx4_ib_query_device;  in mlx4_ib_add()
2233 	ibdev->ib_dev.query_port	= mlx4_ib_query_port;  in mlx4_ib_add()
2234 	ibdev->ib_dev.get_link_layer	= mlx4_ib_port_link_layer;  in mlx4_ib_add()
2235 	ibdev->ib_dev.query_gid		= mlx4_ib_query_gid;  in mlx4_ib_add()
2236 	ibdev->ib_dev.query_pkey	= mlx4_ib_query_pkey;  in mlx4_ib_add()
2237 	ibdev->ib_dev.modify_device	= mlx4_ib_modify_device;  in mlx4_ib_add()
2238 	ibdev->ib_dev.modify_port	= mlx4_ib_modify_port;  in mlx4_ib_add()
2239 	ibdev->ib_dev.alloc_ucontext	= mlx4_ib_alloc_ucontext;  in mlx4_ib_add()
2240 	ibdev->ib_dev.dealloc_ucontext	= mlx4_ib_dealloc_ucontext;  in mlx4_ib_add()
2241 	ibdev->ib_dev.mmap		= mlx4_ib_mmap;  in mlx4_ib_add()
2242 	ibdev->ib_dev.alloc_pd		= mlx4_ib_alloc_pd;  in mlx4_ib_add()
2243 	ibdev->ib_dev.dealloc_pd	= mlx4_ib_dealloc_pd;  in mlx4_ib_add()
2244 	ibdev->ib_dev.create_ah		= mlx4_ib_create_ah;  in mlx4_ib_add()
2245 	ibdev->ib_dev.query_ah		= mlx4_ib_query_ah;  in mlx4_ib_add()
2246 	ibdev->ib_dev.destroy_ah	= mlx4_ib_destroy_ah;  in mlx4_ib_add()
2247 	ibdev->ib_dev.create_srq	= mlx4_ib_create_srq;  in mlx4_ib_add()
2248 	ibdev->ib_dev.modify_srq	= mlx4_ib_modify_srq;  in mlx4_ib_add()
2249 	ibdev->ib_dev.query_srq		= mlx4_ib_query_srq;  in mlx4_ib_add()
2250 	ibdev->ib_dev.destroy_srq	= mlx4_ib_destroy_srq;  in mlx4_ib_add()
2251 	ibdev->ib_dev.post_srq_recv	= mlx4_ib_post_srq_recv;  in mlx4_ib_add()
2252 	ibdev->ib_dev.create_qp		= mlx4_ib_create_qp;  in mlx4_ib_add()
2253 	ibdev->ib_dev.modify_qp		= mlx4_ib_modify_qp;  in mlx4_ib_add()
2254 	ibdev->ib_dev.query_qp		= mlx4_ib_query_qp;  in mlx4_ib_add()
2255 	ibdev->ib_dev.destroy_qp	= mlx4_ib_destroy_qp;  in mlx4_ib_add()
2256 	ibdev->ib_dev.post_send		= mlx4_ib_post_send;  in mlx4_ib_add()
2257 	ibdev->ib_dev.post_recv		= mlx4_ib_post_recv;  in mlx4_ib_add()
2258 	ibdev->ib_dev.create_cq		= mlx4_ib_create_cq;  in mlx4_ib_add()
2259 	ibdev->ib_dev.modify_cq		= mlx4_ib_modify_cq;  in mlx4_ib_add()
2260 	ibdev->ib_dev.resize_cq		= mlx4_ib_resize_cq;  in mlx4_ib_add()
2261 	ibdev->ib_dev.destroy_cq	= mlx4_ib_destroy_cq;  in mlx4_ib_add()
2262 	ibdev->ib_dev.poll_cq		= mlx4_ib_poll_cq;  in mlx4_ib_add()
2263 	ibdev->ib_dev.req_notify_cq	= mlx4_ib_arm_cq;  in mlx4_ib_add()
2264 	ibdev->ib_dev.get_dma_mr	= mlx4_ib_get_dma_mr;  in mlx4_ib_add()
2265 	ibdev->ib_dev.reg_user_mr	= mlx4_ib_reg_user_mr;  in mlx4_ib_add()
2266 	ibdev->ib_dev.rereg_user_mr	= mlx4_ib_rereg_user_mr;  in mlx4_ib_add()
2267 	ibdev->ib_dev.dereg_mr		= mlx4_ib_dereg_mr;  in mlx4_ib_add()
2268 	ibdev->ib_dev.alloc_mr		= mlx4_ib_alloc_mr;  in mlx4_ib_add()
2269 	ibdev->ib_dev.map_mr_sg		= mlx4_ib_map_mr_sg;  in mlx4_ib_add()
2270 	ibdev->ib_dev.attach_mcast	= mlx4_ib_mcg_attach;  in mlx4_ib_add()
2271 	ibdev->ib_dev.detach_mcast	= mlx4_ib_mcg_detach;  in mlx4_ib_add()
2272 	ibdev->ib_dev.process_mad	= mlx4_ib_process_mad;  in mlx4_ib_add()
2273 	ibdev->ib_dev.get_port_immutable = mlx4_port_immutable;  in mlx4_ib_add()
2274 	ibdev->ib_dev.disassociate_ucontext = mlx4_ib_disassociate_ucontext;  in mlx4_ib_add()
2276 	if (!mlx4_is_slave(ibdev->dev)) {  in mlx4_ib_add()
2277 		ibdev->ib_dev.alloc_fmr		= mlx4_ib_fmr_alloc;  in mlx4_ib_add()
2278 		ibdev->ib_dev.map_phys_fmr	= mlx4_ib_map_phys_fmr;  in mlx4_ib_add()
2279 		ibdev->ib_dev.unmap_fmr		= mlx4_ib_unmap_fmr;  in mlx4_ib_add()
2280 		ibdev->ib_dev.dealloc_fmr	= mlx4_ib_fmr_dealloc;  in mlx4_ib_add()
2285 		ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw;  in mlx4_ib_add()
2286 		ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw;  in mlx4_ib_add()
2287 		ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw;  in mlx4_ib_add()
2289 		ibdev->ib_dev.uverbs_cmd_mask |=  in mlx4_ib_add()
2295 		ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd;  in mlx4_ib_add()
2296 		ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd;  in mlx4_ib_add()
2297 		ibdev->ib_dev.uverbs_cmd_mask |=  in mlx4_ib_add()
2303 		ibdev->steering_support = MLX4_STEERING_MODE_DEVICE_MANAGED;  in mlx4_ib_add()
2304 		ibdev->ib_dev.create_flow	= mlx4_ib_create_flow;  in mlx4_ib_add()
2305 		ibdev->ib_dev.destroy_flow	= mlx4_ib_destroy_flow;  in mlx4_ib_add()
2307 		ibdev->ib_dev.uverbs_ex_cmd_mask	|=  in mlx4_ib_add()
2312 	ibdev->ib_dev.uverbs_ex_cmd_mask |=  in mlx4_ib_add()
2317 	mlx4_ib_alloc_eqs(dev, ibdev);  in mlx4_ib_add()
2321 	if (init_node_data(ibdev))  in mlx4_ib_add()
2324 	for (i = 0; i < ibdev->num_ports; ++i) {  in mlx4_ib_add()
2325 		mutex_init(&ibdev->counters_table[i].mutex);  in mlx4_ib_add()
2326 		INIT_LIST_HEAD(&ibdev->counters_table[i].counters_list);  in mlx4_ib_add()
2329 	num_req_counters = mlx4_is_bonded(dev) ? 1 : ibdev->num_ports;  in mlx4_ib_add()
2331 		mutex_init(&ibdev->qp1_proxy_lock[i]);  in mlx4_ib_add()
2333 		if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==  in mlx4_ib_add()
2335 			err = mlx4_counter_alloc(ibdev->dev, &counter_index);  in mlx4_ib_add()
2351 				mlx4_counter_free(ibdev->dev, counter_index);  in mlx4_ib_add()
2357 			      &ibdev->counters_table[i].counters_list);  in mlx4_ib_add()
2358 		ibdev->counters_table[i].default_counter = counter_index;  in mlx4_ib_add()
2363 		for (i = 1; i < ibdev->num_ports ; ++i) {  in mlx4_ib_add()
2372 				      &ibdev->counters_table[i].counters_list);  in mlx4_ib_add()
2373 			ibdev->counters_table[i].default_counter =  in mlx4_ib_add()
2380 	spin_lock_init(&ibdev->sm_lock);  in mlx4_ib_add()
2381 	mutex_init(&ibdev->cap_mask_mutex);  in mlx4_ib_add()
2382 	INIT_LIST_HEAD(&ibdev->qp_list);  in mlx4_ib_add()
2383 	spin_lock_init(&ibdev->reset_flow_resource_lock);  in mlx4_ib_add()
2385 	if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED &&  in mlx4_ib_add()
2387 		ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;  in mlx4_ib_add()
2388 		err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,  in mlx4_ib_add()
2390 					    &ibdev->steer_qpn_base, 0);  in mlx4_ib_add()
2394 		ibdev->ib_uc_qpns_bitmap =  in mlx4_ib_add()
2395 			kmalloc(BITS_TO_LONGS(ibdev->steer_qpn_count) *  in mlx4_ib_add()
2398 		if (!ibdev->ib_uc_qpns_bitmap) {  in mlx4_ib_add()
2404 		bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count);  in mlx4_ib_add()
2407 				dev, ibdev->steer_qpn_base,  in mlx4_ib_add()
2408 				ibdev->steer_qpn_base +  in mlx4_ib_add()
2409 				ibdev->steer_qpn_count - 1);  in mlx4_ib_add()
2414 	for (j = 1; j <= ibdev->dev->caps.num_ports; j++)  in mlx4_ib_add()
2415 		atomic64_set(&iboe->mac[j - 1], ibdev->dev->caps.def_mac[j]);  in mlx4_ib_add()
2417 	if (ib_register_device(&ibdev->ib_dev, NULL))  in mlx4_ib_add()
2420 	if (mlx4_ib_mad_init(ibdev))  in mlx4_ib_add()
2423 	if (mlx4_ib_init_sriov(ibdev))  in mlx4_ib_add()
2438 		if (device_create_file(&ibdev->ib_dev.dev,  in mlx4_ib_add()
2443 	ibdev->ib_active = true;  in mlx4_ib_add()
2445 	if (mlx4_is_mfunc(ibdev->dev))  in mlx4_ib_add()
2446 		init_pkeys(ibdev);  in mlx4_ib_add()
2449 	if (mlx4_is_master(ibdev->dev)) {  in mlx4_ib_add()
2451 			if (j == mlx4_master_func_num(ibdev->dev))  in mlx4_ib_add()
2453 			if (mlx4_is_slave_active(ibdev->dev, j))  in mlx4_ib_add()
2454 				do_slave_init(ibdev, j, 1);  in mlx4_ib_add()
2457 	return ibdev;  in mlx4_ib_add()
2460 	if (ibdev->iboe.nb.notifier_call) {  in mlx4_ib_add()
2461 		if (unregister_netdevice_notifier(&ibdev->iboe.nb))  in mlx4_ib_add()
2463 		ibdev->iboe.nb.notifier_call = NULL;  in mlx4_ib_add()
2467 	mlx4_ib_close_sriov(ibdev);  in mlx4_ib_add()
2470 	mlx4_ib_mad_cleanup(ibdev);  in mlx4_ib_add()
2473 	ib_unregister_device(&ibdev->ib_dev);  in mlx4_ib_add()
2476 	kfree(ibdev->ib_uc_qpns_bitmap);  in mlx4_ib_add()
2479 	if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED)  in mlx4_ib_add()
2480 		mlx4_qp_release_range(dev, ibdev->steer_qpn_base,  in mlx4_ib_add()
2481 				      ibdev->steer_qpn_count);  in mlx4_ib_add()
2483 	for (i = 0; i < ibdev->num_ports; ++i)  in mlx4_ib_add()
2484 		mlx4_ib_delete_counters_table(ibdev, &ibdev->counters_table[i]);  in mlx4_ib_add()
2487 	iounmap(ibdev->uar_map);  in mlx4_ib_add()
2490 	mlx4_uar_free(dev, &ibdev->priv_uar);  in mlx4_ib_add()
2493 	mlx4_pd_free(dev, ibdev->priv_pdn);  in mlx4_ib_add()
2496 	ib_dealloc_device(&ibdev->ib_dev);  in mlx4_ib_add()
2566 	struct mlx4_ib_dev *ibdev = ibdev_ptr;  in mlx4_ib_remove()  local
2569 	ibdev->ib_active = false;  in mlx4_ib_remove()
2572 	mlx4_ib_close_sriov(ibdev);  in mlx4_ib_remove()
2573 	mlx4_ib_mad_cleanup(ibdev);  in mlx4_ib_remove()
2574 	ib_unregister_device(&ibdev->ib_dev);  in mlx4_ib_remove()
2575 	if (ibdev->iboe.nb.notifier_call) {  in mlx4_ib_remove()
2576 		if (unregister_netdevice_notifier(&ibdev->iboe.nb))  in mlx4_ib_remove()
2578 		ibdev->iboe.nb.notifier_call = NULL;  in mlx4_ib_remove()
2581 	if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED) {  in mlx4_ib_remove()
2582 		mlx4_qp_release_range(dev, ibdev->steer_qpn_base,  in mlx4_ib_remove()
2583 				      ibdev->steer_qpn_count);  in mlx4_ib_remove()
2584 		kfree(ibdev->ib_uc_qpns_bitmap);  in mlx4_ib_remove()
2587 	iounmap(ibdev->uar_map);  in mlx4_ib_remove()
2588 	for (p = 0; p < ibdev->num_ports; ++p)  in mlx4_ib_remove()
2589 		mlx4_ib_delete_counters_table(ibdev, &ibdev->counters_table[p]);  in mlx4_ib_remove()
2594 	mlx4_ib_free_eqs(dev, ibdev);  in mlx4_ib_remove()
2596 	mlx4_uar_free(dev, &ibdev->priv_uar);  in mlx4_ib_remove()
2597 	mlx4_pd_free(dev, ibdev->priv_pdn);  in mlx4_ib_remove()
2598 	ib_dealloc_device(&ibdev->ib_dev);  in mlx4_ib_remove()
2601 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)  in do_slave_init()  argument
2604 	struct mlx4_dev *dev = ibdev->dev;  in do_slave_init()
2636 		dm[i]->dev = ibdev;  in do_slave_init()
2639 	spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags);  in do_slave_init()
2640 	if (!ibdev->sriov.is_going_down) {  in do_slave_init()
2642 			queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work);  in do_slave_init()
2643 		spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags);  in do_slave_init()
2645 		spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags);  in do_slave_init()
2654 static void mlx4_ib_handle_catas_error(struct mlx4_ib_dev *ibdev)  in mlx4_ib_handle_catas_error()  argument
2668 	spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);  in mlx4_ib_handle_catas_error()
2670 	list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {  in mlx4_ib_handle_catas_error()
2711 	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);  in mlx4_ib_handle_catas_error()
2719 	struct mlx4_ib_dev *ibdev = ew->ib_dev;  in handle_bonded_port_state_event()  local
2725 	spin_lock_bh(&ibdev->iboe.lock);  in handle_bonded_port_state_event()
2727 		struct net_device *curr_netdev = ibdev->iboe.netdevs[i];  in handle_bonded_port_state_event()
2741 	spin_unlock_bh(&ibdev->iboe.lock);  in handle_bonded_port_state_event()
2743 	ibev.device = &ibdev->ib_dev;  in handle_bonded_port_state_event()
2755 	struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);  in mlx4_ib_event()  local
2767 		ew->ib_dev = ibdev;  in mlx4_ib_event()
2779 		if (p > ibdev->num_ports)  in mlx4_ib_event()
2782 		    rdma_port_get_link_layer(&ibdev->ib_dev, p) ==  in mlx4_ib_event()
2784 			mlx4_ib_invalidate_all_guid_record(ibdev, p);  in mlx4_ib_event()
2790 		if (p > ibdev->num_ports)  in mlx4_ib_event()
2796 		ibdev->ib_active = false;  in mlx4_ib_event()
2798 		mlx4_ib_handle_catas_error(ibdev);  in mlx4_ib_event()
2810 		ew->ib_dev = ibdev;  in mlx4_ib_event()
2820 		do_slave_init(ibdev, p, 1);  in mlx4_ib_event()
2824 			for (i = 1; i <= ibdev->num_ports; i++) {  in mlx4_ib_event()
2825 				if (rdma_port_get_link_layer(&ibdev->ib_dev, i)  in mlx4_ib_event()
2827 					mlx4_ib_slave_alias_guid_event(ibdev,  in mlx4_ib_event()
2838 			for (i = 1; i <= ibdev->num_ports; i++) {  in mlx4_ib_event()
2839 				if (rdma_port_get_link_layer(&ibdev->ib_dev, i)  in mlx4_ib_event()
2841 					mlx4_ib_slave_alias_guid_event(ibdev,  in mlx4_ib_event()
2847 		do_slave_init(ibdev, p, 0);  in mlx4_ib_event()
2855 	ibev.element.port_num = mlx4_is_bonded(ibdev->dev) ? 1 : (u8)p;  in mlx4_ib_event()