171 int mlx4_check_port_params(struct mlx4_dev *dev,  in mlx4_check_port_params()  argument
176 	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {  in mlx4_check_port_params()
177 		for (i = 0; i < dev->caps.num_ports - 1; i++) {  in mlx4_check_port_params()
179 				mlx4_err(dev, "Only same port types supported on this HCA, aborting\n");  in mlx4_check_port_params()
185 	for (i = 0; i < dev->caps.num_ports; i++) {  in mlx4_check_port_params()
186 		if (!(port_type[i] & dev->caps.supported_type[i+1])) {  in mlx4_check_port_params()
187 			mlx4_err(dev, "Requested port type for port %d is not supported on this HCA\n",  in mlx4_check_port_params()
195 static void mlx4_set_port_mask(struct mlx4_dev *dev)  in mlx4_set_port_mask()  argument
199 	for (i = 1; i <= dev->caps.num_ports; ++i)  in mlx4_set_port_mask()
200 		dev->caps.port_mask[i] = dev->caps.port_type[i];  in mlx4_set_port_mask()
207 static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)  in mlx4_query_func()  argument
212 	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {  in mlx4_query_func()
213 		err = mlx4_QUERY_FUNC(dev, &func, 0);  in mlx4_query_func()
215 			mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");  in mlx4_query_func()
226 static void mlx4_enable_cqe_eqe_stride(struct mlx4_dev *dev)  in mlx4_enable_cqe_eqe_stride()  argument
228 	struct mlx4_caps *dev_cap = &dev->caps;  in mlx4_enable_cqe_eqe_stride()
246 		mlx4_dbg(dev, "Enabling CQE stride cacheLine supported\n");  in mlx4_enable_cqe_eqe_stride()
251 		if (mlx4_is_master(dev))  in mlx4_enable_cqe_eqe_stride()
255 			mlx4_dbg(dev, "Disabling CQE stride, cacheLine size unsupported\n");  in mlx4_enable_cqe_eqe_stride()
261 static int _mlx4_dev_port(struct mlx4_dev *dev, int port,  in _mlx4_dev_port()  argument
264 	dev->caps.vl_cap[port]	    = port_cap->max_vl;  in _mlx4_dev_port()
265 	dev->caps.ib_mtu_cap[port]	    = port_cap->ib_mtu;  in _mlx4_dev_port()
266 	dev->phys_caps.gid_phys_table_len[port]  = port_cap->max_gids;  in _mlx4_dev_port()
267 	dev->phys_caps.pkey_phys_table_len[port] = port_cap->max_pkeys;  in _mlx4_dev_port()
271 	dev->caps.gid_table_len[port]  = port_cap->max_gids;  in _mlx4_dev_port()
272 	dev->caps.pkey_table_len[port] = port_cap->max_pkeys;  in _mlx4_dev_port()
273 	dev->caps.port_width_cap[port] = port_cap->max_port_width;  in _mlx4_dev_port()
274 	dev->caps.eth_mtu_cap[port]    = port_cap->eth_mtu;  in _mlx4_dev_port()
275 	dev->caps.def_mac[port]        = port_cap->def_mac;  in _mlx4_dev_port()
276 	dev->caps.supported_type[port] = port_cap->supported_port_types;  in _mlx4_dev_port()
277 	dev->caps.suggested_type[port] = port_cap->suggested_type;  in _mlx4_dev_port()
278 	dev->caps.default_sense[port] = port_cap->default_sense;  in _mlx4_dev_port()
279 	dev->caps.trans_type[port]	    = port_cap->trans_type;  in _mlx4_dev_port()
280 	dev->caps.vendor_oui[port]     = port_cap->vendor_oui;  in _mlx4_dev_port()
281 	dev->caps.wavelength[port]     = port_cap->wavelength;  in _mlx4_dev_port()
282 	dev->caps.trans_code[port]     = port_cap->trans_code;  in _mlx4_dev_port()
287 static int mlx4_dev_port(struct mlx4_dev *dev, int port,  in mlx4_dev_port()  argument
292 	err = mlx4_QUERY_PORT(dev, port, port_cap);  in mlx4_dev_port()
295 		mlx4_err(dev, "QUERY_PORT command failed.\n");  in mlx4_dev_port()
300 static inline void mlx4_enable_ignore_fcs(struct mlx4_dev *dev)  in mlx4_enable_ignore_fcs()  argument
302 	if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS))  in mlx4_enable_ignore_fcs()
305 	if (mlx4_is_mfunc(dev)) {  in mlx4_enable_ignore_fcs()
306 		mlx4_dbg(dev, "SRIOV mode - Disabling Ignore FCS");  in mlx4_enable_ignore_fcs()
307 		dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS;  in mlx4_enable_ignore_fcs()
311 	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) {  in mlx4_enable_ignore_fcs()
312 		mlx4_dbg(dev,  in mlx4_enable_ignore_fcs()
314 		dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS;  in mlx4_enable_ignore_fcs()
320 static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)  in mlx4_dev_cap()  argument
325 	err = mlx4_QUERY_DEV_CAP(dev, dev_cap);  in mlx4_dev_cap()
327 		mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");  in mlx4_dev_cap()
330 	mlx4_dev_cap_dump(dev, dev_cap);  in mlx4_dev_cap()
333 		mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",  in mlx4_dev_cap()
338 		mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",  in mlx4_dev_cap()
343 	if (dev_cap->uar_size > pci_resource_len(dev->persist->pdev, 2)) {  in mlx4_dev_cap()
344 …mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n…  in mlx4_dev_cap()
347 			 pci_resource_len(dev->persist->pdev, 2));  in mlx4_dev_cap()
351 	dev->caps.num_ports	     = dev_cap->num_ports;  in mlx4_dev_cap()
352 	dev->caps.num_sys_eqs = dev_cap->num_sys_eqs;  in mlx4_dev_cap()
353 	dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ?  in mlx4_dev_cap()
354 				      dev->caps.num_sys_eqs :  in mlx4_dev_cap()
356 	for (i = 1; i <= dev->caps.num_ports; ++i) {  in mlx4_dev_cap()
357 		err = _mlx4_dev_port(dev, i, dev_cap->port_cap + i);  in mlx4_dev_cap()
359 			mlx4_err(dev, "QUERY_PORT command failed, aborting\n");  in mlx4_dev_cap()
364 	dev->caps.uar_page_size	     = PAGE_SIZE;  in mlx4_dev_cap()
365 	dev->caps.num_uars	     = dev_cap->uar_size / PAGE_SIZE;  in mlx4_dev_cap()
366 	dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;  in mlx4_dev_cap()
367 	dev->caps.bf_reg_size	     = dev_cap->bf_reg_size;  in mlx4_dev_cap()
368 	dev->caps.bf_regs_per_page   = dev_cap->bf_regs_per_page;  in mlx4_dev_cap()
369 	dev->caps.max_sq_sg	     = dev_cap->max_sq_sg;  in mlx4_dev_cap()
370 	dev->caps.max_rq_sg	     = dev_cap->max_rq_sg;  in mlx4_dev_cap()
371 	dev->caps.max_wqes	     = dev_cap->max_qp_sz;  in mlx4_dev_cap()
372 	dev->caps.max_qp_init_rdma   = dev_cap->max_requester_per_qp;  in mlx4_dev_cap()
373 	dev->caps.max_srq_wqes	     = dev_cap->max_srq_sz;  in mlx4_dev_cap()
374 	dev->caps.max_srq_sge	     = dev_cap->max_rq_sg - 1;  in mlx4_dev_cap()
375 	dev->caps.reserved_srqs	     = dev_cap->reserved_srqs;  in mlx4_dev_cap()
376 	dev->caps.max_sq_desc_sz     = dev_cap->max_sq_desc_sz;  in mlx4_dev_cap()
377 	dev->caps.max_rq_desc_sz     = dev_cap->max_rq_desc_sz;  in mlx4_dev_cap()
383 	dev->caps.max_cqes	     = dev_cap->max_cq_sz - 1;  in mlx4_dev_cap()
384 	dev->caps.reserved_cqs	     = dev_cap->reserved_cqs;  in mlx4_dev_cap()
385 	dev->caps.reserved_eqs	     = dev_cap->reserved_eqs;  in mlx4_dev_cap()
386 	dev->caps.reserved_mtts      = dev_cap->reserved_mtts;  in mlx4_dev_cap()
387 	dev->caps.reserved_mrws	     = dev_cap->reserved_mrws;  in mlx4_dev_cap()
390 	dev->caps.reserved_uars	     = max_t(int, 128, dev_cap->reserved_uars);  in mlx4_dev_cap()
391 	dev->caps.reserved_pds	     = dev_cap->reserved_pds;  in mlx4_dev_cap()
392 	dev->caps.reserved_xrcds     = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?  in mlx4_dev_cap()
394 	dev->caps.max_xrcds          = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?  in mlx4_dev_cap()
396 	dev->caps.mtt_entry_sz       = dev_cap->mtt_entry_sz;  in mlx4_dev_cap()
398 	dev->caps.max_msg_sz         = dev_cap->max_msg_sz;  in mlx4_dev_cap()
399 	dev->caps.page_size_cap	     = ~(u32) (dev_cap->min_page_sz - 1);  in mlx4_dev_cap()
400 	dev->caps.flags		     = dev_cap->flags;  in mlx4_dev_cap()
401 	dev->caps.flags2	     = dev_cap->flags2;  in mlx4_dev_cap()
402 	dev->caps.bmme_flags	     = dev_cap->bmme_flags;  in mlx4_dev_cap()
403 	dev->caps.reserved_lkey	     = dev_cap->reserved_lkey;  in mlx4_dev_cap()
404 	dev->caps.stat_rate_support  = dev_cap->stat_rate_support;  in mlx4_dev_cap()
405 	dev->caps.max_gso_sz	     = dev_cap->max_gso_sz;  in mlx4_dev_cap()
406 	dev->caps.max_rss_tbl_sz     = dev_cap->max_rss_tbl_sz;  in mlx4_dev_cap()
408 	if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN) {  in mlx4_dev_cap()
412 		err = mlx4_QUERY_HCA(dev, &hca_param);  in mlx4_dev_cap()
420 			dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_PHV_EN;  in mlx4_dev_cap()
424 	if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT)  in mlx4_dev_cap()
425 		dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;  in mlx4_dev_cap()
427 	if (mlx4_is_mfunc(dev))  in mlx4_dev_cap()
428 		dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;  in mlx4_dev_cap()
431 		dev->caps.log_num_macs  = MLX4_MIN_LOG_NUM_MAC;  in mlx4_dev_cap()
432 		dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS;  in mlx4_dev_cap()
434 		dev->caps.log_num_macs  = log_num_mac;  in mlx4_dev_cap()
435 		dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS;  in mlx4_dev_cap()
438 	for (i = 1; i <= dev->caps.num_ports; ++i) {  in mlx4_dev_cap()
439 		dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE;  in mlx4_dev_cap()
440 		if (dev->caps.supported_type[i]) {  in mlx4_dev_cap()
442 			if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH)  in mlx4_dev_cap()
443 				dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;  in mlx4_dev_cap()
445 			else if (dev->caps.supported_type[i] ==  in mlx4_dev_cap()
447 				dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;  in mlx4_dev_cap()
453 					dev->caps.port_type[i] = dev->caps.suggested_type[i] ?  in mlx4_dev_cap()
456 					dev->caps.port_type[i] = port_type_array[i - 1];  in mlx4_dev_cap()
465 		mlx4_priv(dev)->sense.sense_allowed[i] =  in mlx4_dev_cap()
466 			((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) &&  in mlx4_dev_cap()
467 			 (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&  in mlx4_dev_cap()
468 			 (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT));  in mlx4_dev_cap()
475 		if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) {  in mlx4_dev_cap()
477 			dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO;  in mlx4_dev_cap()
478 			mlx4_SENSE_PORT(dev, i, &sensed_port);  in mlx4_dev_cap()
480 				dev->caps.port_type[i] = sensed_port;  in mlx4_dev_cap()
482 			dev->caps.possible_type[i] = dev->caps.port_type[i];  in mlx4_dev_cap()
485 		if (dev->caps.log_num_macs > dev_cap->port_cap[i].log_max_macs) {  in mlx4_dev_cap()
486 			dev->caps.log_num_macs = dev_cap->port_cap[i].log_max_macs;  in mlx4_dev_cap()
487 			mlx4_warn(dev, "Requested number of MACs is too much for port %d, reducing to %d\n",  in mlx4_dev_cap()
488 				  i, 1 << dev->caps.log_num_macs);  in mlx4_dev_cap()
490 		if (dev->caps.log_num_vlans > dev_cap->port_cap[i].log_max_vlans) {  in mlx4_dev_cap()
491 			dev->caps.log_num_vlans = dev_cap->port_cap[i].log_max_vlans;  in mlx4_dev_cap()
492 			mlx4_warn(dev, "Requested number of VLANs is too much for port %d, reducing to %d\n",  in mlx4_dev_cap()
493 				  i, 1 << dev->caps.log_num_vlans);  in mlx4_dev_cap()
497 	if (mlx4_is_master(dev) && (dev->caps.num_ports == 2) &&  in mlx4_dev_cap()
500 		mlx4_warn(dev,  in mlx4_dev_cap()
502 		dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_QOS_VPP;  in mlx4_dev_cap()
505 	dev->caps.max_counters = dev_cap->max_counters;  in mlx4_dev_cap()
507 	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;  in mlx4_dev_cap()
508 	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =  in mlx4_dev_cap()
509 		dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =  in mlx4_dev_cap()
510 		(1 << dev->caps.log_num_macs) *  in mlx4_dev_cap()
511 		(1 << dev->caps.log_num_vlans) *  in mlx4_dev_cap()
512 		dev->caps.num_ports;  in mlx4_dev_cap()
513 	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;  in mlx4_dev_cap()
516 	    dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN)  in mlx4_dev_cap()
517 		dev->caps.dmfs_high_rate_qpn_base = dev_cap->dmfs_high_rate_qpn_base;  in mlx4_dev_cap()
519 		dev->caps.dmfs_high_rate_qpn_base =  in mlx4_dev_cap()
520 			dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];  in mlx4_dev_cap()
523 	    dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) {  in mlx4_dev_cap()
524 		dev->caps.dmfs_high_rate_qpn_range = dev_cap->dmfs_high_rate_qpn_range;  in mlx4_dev_cap()
525 		dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DEFAULT;  in mlx4_dev_cap()
526 		dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_FS_A0;  in mlx4_dev_cap()
528 		dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_NOT_SUPPORTED;  in mlx4_dev_cap()
529 		dev->caps.dmfs_high_rate_qpn_base =  in mlx4_dev_cap()
530 			dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];  in mlx4_dev_cap()
531 		dev->caps.dmfs_high_rate_qpn_range = MLX4_A0_STEERING_TABLE_SIZE;  in mlx4_dev_cap()
534 	dev->caps.rl_caps = dev_cap->rl_caps;  in mlx4_dev_cap()
536 	dev->caps.reserved_qps_cnt[MLX4_QP_REGION_RSS_RAW_ETH] =  in mlx4_dev_cap()
537 		dev->caps.dmfs_high_rate_qpn_range;  in mlx4_dev_cap()
539 	dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +  in mlx4_dev_cap()
540 		dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +  in mlx4_dev_cap()
541 		dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] +  in mlx4_dev_cap()
542 		dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];  in mlx4_dev_cap()
544 	dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0;  in mlx4_dev_cap()
546 	if (!enable_64b_cqe_eqe && !mlx4_is_slave(dev)) {  in mlx4_dev_cap()
549 			mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n");  in mlx4_dev_cap()
550 			dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;  in mlx4_dev_cap()
551 			dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;  in mlx4_dev_cap()
557 			mlx4_warn(dev, "Disabling EQE/CQE stride per user request\n");  in mlx4_dev_cap()
563 	if ((dev->caps.flags &  in mlx4_dev_cap()
565 	    mlx4_is_master(dev))  in mlx4_dev_cap()
566 		dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE;  in mlx4_dev_cap()
568 	if (!mlx4_is_slave(dev)) {  in mlx4_dev_cap()
569 		mlx4_enable_cqe_eqe_stride(dev);  in mlx4_dev_cap()
570 		dev->caps.alloc_res_qp_mask =  in mlx4_dev_cap()
571 			(dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0) |  in mlx4_dev_cap()
574 		if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) &&  in mlx4_dev_cap()
575 		    dev->caps.flags & MLX4_DEV_CAP_FLAG_SET_ETH_SCHED) {  in mlx4_dev_cap()
576 			mlx4_warn(dev, "Old device ETS support detected\n");  in mlx4_dev_cap()
577 			mlx4_warn(dev, "Consider upgrading device FW.\n");  in mlx4_dev_cap()
578 			dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG;  in mlx4_dev_cap()
582 		dev->caps.alloc_res_qp_mask = 0;  in mlx4_dev_cap()
585 	mlx4_enable_ignore_fcs(dev);  in mlx4_dev_cap()
590 static int mlx4_get_pcie_dev_link_caps(struct mlx4_dev *dev,  in mlx4_get_pcie_dev_link_caps()  argument
602 	err1 = pcie_capability_read_dword(dev->persist->pdev, PCI_EXP_LNKCAP,  in mlx4_get_pcie_dev_link_caps()
604 	err2 = pcie_capability_read_dword(dev->persist->pdev, PCI_EXP_LNKCAP2,  in mlx4_get_pcie_dev_link_caps()
631 static void mlx4_check_pcie_caps(struct mlx4_dev *dev)  in mlx4_check_pcie_caps()  argument
643 	err = mlx4_get_pcie_dev_link_caps(dev, &speed_cap, &width_cap);  in mlx4_check_pcie_caps()
645 		mlx4_warn(dev,  in mlx4_check_pcie_caps()
650 	err = pcie_get_minimum_link(dev->persist->pdev, &speed, &width);  in mlx4_check_pcie_caps()
653 		mlx4_warn(dev,  in mlx4_check_pcie_caps()
659 		mlx4_warn(dev,  in mlx4_check_pcie_caps()
662 	mlx4_info(dev, "PCIe link speed is %s, device supports %s\n",  in mlx4_check_pcie_caps()
664 	mlx4_info(dev, "PCIe link width is x%d, device supports x%d\n",  in mlx4_check_pcie_caps()
670 static int mlx4_how_many_lives_vf(struct mlx4_dev *dev)  in mlx4_how_many_lives_vf()  argument
672 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_how_many_lives_vf()
677 	for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) {  in mlx4_how_many_lives_vf()
681 			mlx4_warn(dev, "%s: slave: %d is still active\n",  in mlx4_how_many_lives_vf()
689 int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey)  in mlx4_get_parav_qkey()  argument
693 	if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX ||  in mlx4_get_parav_qkey()
694 	    qpn < dev->phys_caps.base_proxy_sqpn)  in mlx4_get_parav_qkey()
697 	if (qpn >= dev->phys_caps.base_tunnel_sqpn)  in mlx4_get_parav_qkey()
699 		qk += qpn - dev->phys_caps.base_tunnel_sqpn;  in mlx4_get_parav_qkey()
701 		qk += qpn - dev->phys_caps.base_proxy_sqpn;  in mlx4_get_parav_qkey()
707 void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val)  in mlx4_sync_pkey_table()  argument
709 	struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);  in mlx4_sync_pkey_table()
711 	if (!mlx4_is_master(dev))  in mlx4_sync_pkey_table()
718 void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid)  in mlx4_put_slave_node_guid()  argument
720 	struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);  in mlx4_put_slave_node_guid()
722 	if (!mlx4_is_master(dev))  in mlx4_put_slave_node_guid()
729 __be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave)  in mlx4_get_slave_node_guid()  argument
731 	struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);  in mlx4_get_slave_node_guid()
733 	if (!mlx4_is_master(dev))  in mlx4_get_slave_node_guid()
740 int mlx4_is_slave_active(struct mlx4_dev *dev, int slave)  in mlx4_is_slave_active()  argument
742 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_is_slave_active()
745 	if (!mlx4_is_master(dev))  in mlx4_is_slave_active()
753 static void slave_adjust_steering_mode(struct mlx4_dev *dev,  in slave_adjust_steering_mode()  argument
757 	dev->caps.steering_mode = hca_param->steering_mode;  in slave_adjust_steering_mode()
758 	if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {  in slave_adjust_steering_mode()
759 		dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;  in slave_adjust_steering_mode()
760 		dev->caps.fs_log_max_ucast_qp_range_size =  in slave_adjust_steering_mode()
763 		dev->caps.num_qp_per_mgm =  in slave_adjust_steering_mode()
766 	mlx4_dbg(dev, "Steering mode is: %s\n",  in slave_adjust_steering_mode()
767 		 mlx4_steering_mode_str(dev->caps.steering_mode));  in slave_adjust_steering_mode()
770 static int mlx4_slave_cap(struct mlx4_dev *dev)  in mlx4_slave_cap()  argument
780 	err = mlx4_QUERY_HCA(dev, &hca_param);  in mlx4_slave_cap()
782 		mlx4_err(dev, "QUERY_HCA command failed, aborting\n");  in mlx4_slave_cap()
790 		mlx4_err(dev, "Unknown hca global capabilities\n");  in mlx4_slave_cap()
796 	dev->caps.hca_core_clock = hca_param.hca_core_clock;  in mlx4_slave_cap()
799 	dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp;  in mlx4_slave_cap()
800 	err = mlx4_dev_cap(dev, &dev_cap);  in mlx4_slave_cap()
802 		mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");  in mlx4_slave_cap()
806 	err = mlx4_QUERY_FW(dev);  in mlx4_slave_cap()
808 		mlx4_err(dev, "QUERY_FW command failed: could not get FW version\n");  in mlx4_slave_cap()
810 	page_size = ~dev->caps.page_size_cap + 1;  in mlx4_slave_cap()
811 	mlx4_warn(dev, "HCA minimum page size:%d\n", page_size);  in mlx4_slave_cap()
813 		mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",  in mlx4_slave_cap()
819 	dev->caps.uar_page_size = 1 << (hca_param.uar_page_sz + 12);  in mlx4_slave_cap()
822 	if (dev->caps.uar_page_size != PAGE_SIZE) {  in mlx4_slave_cap()
823 		mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %ld\n",  in mlx4_slave_cap()
824 			 dev->caps.uar_page_size, PAGE_SIZE);  in mlx4_slave_cap()
829 	err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap);  in mlx4_slave_cap()
831 		mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n",  in mlx4_slave_cap()
838 		mlx4_err(dev, "Unknown pf context behaviour %x known flags %x\n",  in mlx4_slave_cap()
843 	dev->caps.num_ports		= func_cap.num_ports;  in mlx4_slave_cap()
844 	dev->quotas.qp			= func_cap.qp_quota;  in mlx4_slave_cap()
845 	dev->quotas.srq			= func_cap.srq_quota;  in mlx4_slave_cap()
846 	dev->quotas.cq			= func_cap.cq_quota;  in mlx4_slave_cap()
847 	dev->quotas.mpt			= func_cap.mpt_quota;  in mlx4_slave_cap()
848 	dev->quotas.mtt			= func_cap.mtt_quota;  in mlx4_slave_cap()
849 	dev->caps.num_qps		= 1 << hca_param.log_num_qps;  in mlx4_slave_cap()
850 	dev->caps.num_srqs		= 1 << hca_param.log_num_srqs;  in mlx4_slave_cap()
851 	dev->caps.num_cqs		= 1 << hca_param.log_num_cqs;  in mlx4_slave_cap()
852 	dev->caps.num_mpts		= 1 << hca_param.log_mpt_sz;  in mlx4_slave_cap()
853 	dev->caps.num_eqs		= func_cap.max_eq;  in mlx4_slave_cap()
854 	dev->caps.reserved_eqs		= func_cap.reserved_eq;  in mlx4_slave_cap()
855 	dev->caps.reserved_lkey		= func_cap.reserved_lkey;  in mlx4_slave_cap()
856 	dev->caps.num_pds               = MLX4_NUM_PDS;  in mlx4_slave_cap()
857 	dev->caps.num_mgms              = 0;  in mlx4_slave_cap()
858 	dev->caps.num_amgms             = 0;  in mlx4_slave_cap()
860 	if (dev->caps.num_ports > MLX4_MAX_PORTS) {  in mlx4_slave_cap()
861 		mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",  in mlx4_slave_cap()
862 			 dev->caps.num_ports, MLX4_MAX_PORTS);  in mlx4_slave_cap()
866 	mlx4_replace_zero_macs(dev);  in mlx4_slave_cap()
868 	dev->caps.qp0_qkey = kcalloc(dev->caps.num_ports, sizeof(u32), GFP_KERNEL);  in mlx4_slave_cap()
869 	dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);  in mlx4_slave_cap()
870 	dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);  in mlx4_slave_cap()
871 	dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);  in mlx4_slave_cap()
872 	dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);  in mlx4_slave_cap()
874 	if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||  in mlx4_slave_cap()
875 	    !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy ||  in mlx4_slave_cap()
876 	    !dev->caps.qp0_qkey) {  in mlx4_slave_cap()
881 	for (i = 1; i <= dev->caps.num_ports; ++i) {  in mlx4_slave_cap()
882 		err = mlx4_QUERY_FUNC_CAP(dev, i, &func_cap);  in mlx4_slave_cap()
884 			mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n",  in mlx4_slave_cap()
888 		dev->caps.qp0_qkey[i - 1] = func_cap.qp0_qkey;  in mlx4_slave_cap()
889 		dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn;  in mlx4_slave_cap()
890 		dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn;  in mlx4_slave_cap()
891 		dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn;  in mlx4_slave_cap()
892 		dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn;  in mlx4_slave_cap()
893 		dev->caps.port_mask[i] = dev->caps.port_type[i];  in mlx4_slave_cap()
894 		dev->caps.phys_port_id[i] = func_cap.phys_port_id;  in mlx4_slave_cap()
895 		err = mlx4_get_slave_pkey_gid_tbl_len(dev, i,  in mlx4_slave_cap()
896 						      &dev->caps.gid_table_len[i],  in mlx4_slave_cap()
897 						      &dev->caps.pkey_table_len[i]);  in mlx4_slave_cap()
902 	if (dev->caps.uar_page_size * (dev->caps.num_uars -  in mlx4_slave_cap()
903 				       dev->caps.reserved_uars) >  in mlx4_slave_cap()
904 				       pci_resource_len(dev->persist->pdev,  in mlx4_slave_cap()
906 …mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than PCI resource 2 size of 0x%llx, abo…  in mlx4_slave_cap()
907 			 dev->caps.uar_page_size * dev->caps.num_uars,  in mlx4_slave_cap()
909 			 pci_resource_len(dev->persist->pdev, 2));  in mlx4_slave_cap()
915 		dev->caps.eqe_size   = 64;  in mlx4_slave_cap()
916 		dev->caps.eqe_factor = 1;  in mlx4_slave_cap()
918 		dev->caps.eqe_size   = 32;  in mlx4_slave_cap()
919 		dev->caps.eqe_factor = 0;  in mlx4_slave_cap()
923 		dev->caps.cqe_size   = 64;  in mlx4_slave_cap()
924 		dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;  in mlx4_slave_cap()
926 		dev->caps.cqe_size   = 32;  in mlx4_slave_cap()
930 		dev->caps.eqe_size = hca_param.eqe_size;  in mlx4_slave_cap()
931 		dev->caps.eqe_factor = 0;  in mlx4_slave_cap()
935 		dev->caps.cqe_size = hca_param.cqe_size;  in mlx4_slave_cap()
937 		dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;  in mlx4_slave_cap()
940 	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;  in mlx4_slave_cap()
941 	mlx4_warn(dev, "Timestamping is not supported in slave mode\n");  in mlx4_slave_cap()
943 	slave_adjust_steering_mode(dev, &dev_cap, &hca_param);  in mlx4_slave_cap()
944 	mlx4_dbg(dev, "RSS support for IP fragments is %s\n",  in mlx4_slave_cap()
948 	    dev->caps.bf_reg_size)  in mlx4_slave_cap()
949 		dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP;  in mlx4_slave_cap()
952 		dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP;  in mlx4_slave_cap()
957 	kfree(dev->caps.qp0_qkey);  in mlx4_slave_cap()
958 	kfree(dev->caps.qp0_tunnel);  in mlx4_slave_cap()
959 	kfree(dev->caps.qp0_proxy);  in mlx4_slave_cap()
960 	kfree(dev->caps.qp1_tunnel);  in mlx4_slave_cap()
961 	kfree(dev->caps.qp1_proxy);  in mlx4_slave_cap()
962 	dev->caps.qp0_qkey = NULL;  in mlx4_slave_cap()
963 	dev->caps.qp0_tunnel = NULL;  in mlx4_slave_cap()
964 	dev->caps.qp0_proxy = NULL;  in mlx4_slave_cap()
965 	dev->caps.qp1_tunnel = NULL;  in mlx4_slave_cap()
966 	dev->caps.qp1_proxy = NULL;  in mlx4_slave_cap()
971 static void mlx4_request_modules(struct mlx4_dev *dev)  in mlx4_request_modules()  argument
979 	for (port = 1; port <= dev->caps.num_ports; port++) {  in mlx4_request_modules()
980 		if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB)  in mlx4_request_modules()
982 		else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)  in mlx4_request_modules()
988 	if (has_ib_port || (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE))  in mlx4_request_modules()
996 int mlx4_change_port_types(struct mlx4_dev *dev,  in mlx4_change_port_types()  argument
1003 	for (port = 0; port <  dev->caps.num_ports; port++) {  in mlx4_change_port_types()
1006 		if (port_types[port] != dev->caps.port_type[port + 1])  in mlx4_change_port_types()
1010 		mlx4_unregister_device(dev);  in mlx4_change_port_types()
1011 		for (port = 1; port <= dev->caps.num_ports; port++) {  in mlx4_change_port_types()
1012 			mlx4_CLOSE_PORT(dev, port);  in mlx4_change_port_types()
1013 			dev->caps.port_type[port] = port_types[port - 1];  in mlx4_change_port_types()
1014 			err = mlx4_SET_PORT(dev, port, -1);  in mlx4_change_port_types()
1016 				mlx4_err(dev, "Failed to set port %d, aborting\n",  in mlx4_change_port_types()
1021 		mlx4_set_port_mask(dev);  in mlx4_change_port_types()
1022 		err = mlx4_register_device(dev);  in mlx4_change_port_types()
1024 			mlx4_err(dev, "Failed to register device\n");  in mlx4_change_port_types()
1027 		mlx4_request_modules(dev);  in mlx4_change_port_types()
1034 static ssize_t show_port_type(struct device *dev,  in show_port_type()  argument
1040 	struct mlx4_dev *mdev = info->dev;  in show_port_type()
1054 static ssize_t set_port_type(struct device *dev,  in set_port_type()  argument
1060 	struct mlx4_dev *mdev = info->dev;  in set_port_type()
1163 static ssize_t show_port_ib_mtu(struct device *dev,  in show_port_ib_mtu()  argument
1169 	struct mlx4_dev *mdev = info->dev;  in show_port_ib_mtu()
1179 static ssize_t set_port_ib_mtu(struct device *dev,  in set_port_ib_mtu()  argument
1185 	struct mlx4_dev *mdev = info->dev;  in set_port_ib_mtu()
1224 int mlx4_bond(struct mlx4_dev *dev)  in mlx4_bond()  argument
1227 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_bond()
1231 	if (!mlx4_is_bonded(dev))  in mlx4_bond()
1232 		ret = mlx4_do_bond(dev, true);  in mlx4_bond()
1238 		mlx4_err(dev, "Failed to bond device: %d\n", ret);  in mlx4_bond()
1240 		mlx4_dbg(dev, "Device is bonded\n");  in mlx4_bond()
1245 int mlx4_unbond(struct mlx4_dev *dev)  in mlx4_unbond()  argument
1248 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_unbond()
1252 	if (mlx4_is_bonded(dev))  in mlx4_unbond()
1253 		ret = mlx4_do_bond(dev, false);  in mlx4_unbond()
1257 		mlx4_err(dev, "Failed to unbond device: %d\n", ret);  in mlx4_unbond()
1259 		mlx4_dbg(dev, "Device is unbonded\n");  in mlx4_unbond()
1265 int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p)  in mlx4_port_map_set()  argument
1269 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_port_map_set()
1272 	if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP))  in mlx4_port_map_set()
1293 		err = mlx4_virt2phy_port_map(dev, port1, port2);  in mlx4_port_map_set()
1295 			mlx4_dbg(dev, "port map changed: [%d][%d]\n",  in mlx4_port_map_set()
1300 			mlx4_err(dev, "Failed to change port mape: %d\n", err);  in mlx4_port_map_set()
1309 static int mlx4_load_fw(struct mlx4_dev *dev)  in mlx4_load_fw()  argument
1311 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_load_fw()
1314 	priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages,  in mlx4_load_fw()
1317 		mlx4_err(dev, "Couldn't allocate FW area, aborting\n");  in mlx4_load_fw()
1321 	err = mlx4_MAP_FA(dev, priv->fw.fw_icm);  in mlx4_load_fw()
1323 		mlx4_err(dev, "MAP_FA command failed, aborting\n");  in mlx4_load_fw()
1327 	err = mlx4_RUN_FW(dev);  in mlx4_load_fw()
1329 		mlx4_err(dev, "RUN_FW command failed, aborting\n");  in mlx4_load_fw()
1336 	mlx4_UNMAP_FA(dev);  in mlx4_load_fw()
1339 	mlx4_free_icm(dev, priv->fw.fw_icm, 0);  in mlx4_load_fw()
1343 static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,  in mlx4_init_cmpt_table()  argument
1346 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_init_cmpt_table()
1350 	err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table,  in mlx4_init_cmpt_table()
1354 				  cmpt_entry_sz, dev->caps.num_qps,  in mlx4_init_cmpt_table()
1355 				  dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],  in mlx4_init_cmpt_table()
1360 	err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table,  in mlx4_init_cmpt_table()
1364 				  cmpt_entry_sz, dev->caps.num_srqs,  in mlx4_init_cmpt_table()
1365 				  dev->caps.reserved_srqs, 0, 0);  in mlx4_init_cmpt_table()
1369 	err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table,  in mlx4_init_cmpt_table()
1373 				  cmpt_entry_sz, dev->caps.num_cqs,  in mlx4_init_cmpt_table()
1374 				  dev->caps.reserved_cqs, 0, 0);  in mlx4_init_cmpt_table()
1378 	num_eqs = dev->phys_caps.num_phys_eqs;  in mlx4_init_cmpt_table()
1379 	err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table,  in mlx4_init_cmpt_table()
1390 	mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);  in mlx4_init_cmpt_table()
1393 	mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);  in mlx4_init_cmpt_table()
1396 	mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);  in mlx4_init_cmpt_table()
1402 static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,  in mlx4_init_icm()  argument
1405 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_init_icm()
1410 	err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);  in mlx4_init_icm()
1412 		mlx4_err(dev, "SET_ICM_SIZE command failed, aborting\n");  in mlx4_init_icm()
1416 	mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory\n",  in mlx4_init_icm()
1420 	priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,  in mlx4_init_icm()
1423 		mlx4_err(dev, "Couldn't allocate aux memory, aborting\n");  in mlx4_init_icm()
1427 	err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);  in mlx4_init_icm()
1429 		mlx4_err(dev, "MAP_ICM_AUX command failed, aborting\n");  in mlx4_init_icm()
1433 	err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);  in mlx4_init_icm()
1435 		mlx4_err(dev, "Failed to map cMPT context memory, aborting\n");  in mlx4_init_icm()
1440 	num_eqs = dev->phys_caps.num_phys_eqs;  in mlx4_init_icm()
1441 	err = mlx4_init_icm_table(dev, &priv->eq_table.table,  in mlx4_init_icm()
1445 		mlx4_err(dev, "Failed to map EQ context memory, aborting\n");  in mlx4_init_icm()
1456 	dev->caps.reserved_mtts =  in mlx4_init_icm()
1457 		ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,  in mlx4_init_icm()
1458 		      dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;  in mlx4_init_icm()
1460 	err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,  in mlx4_init_icm()
1462 				  dev->caps.mtt_entry_sz,  in mlx4_init_icm()
1463 				  dev->caps.num_mtts,  in mlx4_init_icm()
1464 				  dev->caps.reserved_mtts, 1, 0);  in mlx4_init_icm()
1466 		mlx4_err(dev, "Failed to map MTT context memory, aborting\n");  in mlx4_init_icm()
1470 	err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,  in mlx4_init_icm()
1473 				  dev->caps.num_mpts,  in mlx4_init_icm()
1474 				  dev->caps.reserved_mrws, 1, 1);  in mlx4_init_icm()
1476 		mlx4_err(dev, "Failed to map dMPT context memory, aborting\n");  in mlx4_init_icm()
1480 	err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,  in mlx4_init_icm()
1483 				  dev->caps.num_qps,  in mlx4_init_icm()
1484 				  dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],  in mlx4_init_icm()
1487 		mlx4_err(dev, "Failed to map QP context memory, aborting\n");  in mlx4_init_icm()
1491 	err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,  in mlx4_init_icm()
1494 				  dev->caps.num_qps,  in mlx4_init_icm()
1495 				  dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],  in mlx4_init_icm()
1498 		mlx4_err(dev, "Failed to map AUXC context memory, aborting\n");  in mlx4_init_icm()
1502 	err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,  in mlx4_init_icm()
1505 				  dev->caps.num_qps,  in mlx4_init_icm()
1506 				  dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],  in mlx4_init_icm()
1509 		mlx4_err(dev, "Failed to map ALTC context memory, aborting\n");  in mlx4_init_icm()
1513 	err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,  in mlx4_init_icm()
1516 				  dev->caps.num_qps,  in mlx4_init_icm()
1517 				  dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],  in mlx4_init_icm()
1520 		mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");  in mlx4_init_icm()
1524 	err = mlx4_init_icm_table(dev, &priv->cq_table.table,  in mlx4_init_icm()
1527 				  dev->caps.num_cqs,  in mlx4_init_icm()
1528 				  dev->caps.reserved_cqs, 0, 0);  in mlx4_init_icm()
1530 		mlx4_err(dev, "Failed to map CQ context memory, aborting\n");  in mlx4_init_icm()
1534 	err = mlx4_init_icm_table(dev, &priv->srq_table.table,  in mlx4_init_icm()
1537 				  dev->caps.num_srqs,  in mlx4_init_icm()
1538 				  dev->caps.reserved_srqs, 0, 0);  in mlx4_init_icm()
1540 		mlx4_err(dev, "Failed to map SRQ context memory, aborting\n");  in mlx4_init_icm()
1551 	err = mlx4_init_icm_table(dev, &priv->mcg_table.table,  in mlx4_init_icm()
1553 				  mlx4_get_mgm_entry_size(dev),  in mlx4_init_icm()
1554 				  dev->caps.num_mgms + dev->caps.num_amgms,  in mlx4_init_icm()
1555 				  dev->caps.num_mgms + dev->caps.num_amgms,  in mlx4_init_icm()
1558 		mlx4_err(dev, "Failed to map MCG context memory, aborting\n");  in mlx4_init_icm()
1565 	mlx4_cleanup_icm_table(dev, &priv->srq_table.table);  in mlx4_init_icm()
1568 	mlx4_cleanup_icm_table(dev, &priv->cq_table.table);  in mlx4_init_icm()
1571 	mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);  in mlx4_init_icm()
1574 	mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);  in mlx4_init_icm()
1577 	mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);  in mlx4_init_icm()
1580 	mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);  in mlx4_init_icm()
1583 	mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);  in mlx4_init_icm()
1586 	mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);  in mlx4_init_icm()
1589 	mlx4_cleanup_icm_table(dev, &priv->eq_table.table);  in mlx4_init_icm()
1592 	mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);  in mlx4_init_icm()
1593 	mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);  in mlx4_init_icm()
1594 	mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);  in mlx4_init_icm()
1595 	mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);  in mlx4_init_icm()
1598 	mlx4_UNMAP_ICM_AUX(dev);  in mlx4_init_icm()
1601 	mlx4_free_icm(dev, priv->fw.aux_icm, 0);  in mlx4_init_icm()
1606 static void mlx4_free_icms(struct mlx4_dev *dev)  in mlx4_free_icms()  argument
1608 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_free_icms()
1610 	mlx4_cleanup_icm_table(dev, &priv->mcg_table.table);  in mlx4_free_icms()
1611 	mlx4_cleanup_icm_table(dev, &priv->srq_table.table);  in mlx4_free_icms()
1612 	mlx4_cleanup_icm_table(dev, &priv->cq_table.table);  in mlx4_free_icms()
1613 	mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);  in mlx4_free_icms()
1614 	mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);  in mlx4_free_icms()
1615 	mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);  in mlx4_free_icms()
1616 	mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);  in mlx4_free_icms()
1617 	mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);  in mlx4_free_icms()
1618 	mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);  in mlx4_free_icms()
1619 	mlx4_cleanup_icm_table(dev, &priv->eq_table.table);  in mlx4_free_icms()
1620 	mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);  in mlx4_free_icms()
1621 	mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);  in mlx4_free_icms()
1622 	mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);  in mlx4_free_icms()
1623 	mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);  in mlx4_free_icms()
1625 	mlx4_UNMAP_ICM_AUX(dev);  in mlx4_free_icms()
1626 	mlx4_free_icm(dev, priv->fw.aux_icm, 0);  in mlx4_free_icms()
1629 static void mlx4_slave_exit(struct mlx4_dev *dev)  in mlx4_slave_exit()  argument
1631 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_slave_exit()
1634 	if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP,  in mlx4_slave_exit()
1636 		mlx4_warn(dev, "Failed to close slave function\n");  in mlx4_slave_exit()
1640 static int map_bf_area(struct mlx4_dev *dev)  in map_bf_area()  argument
1642 	struct mlx4_priv *priv = mlx4_priv(dev);  in map_bf_area()
1647 	if (!dev->caps.bf_reg_size)  in map_bf_area()
1650 	bf_start = pci_resource_start(dev->persist->pdev, 2) +  in map_bf_area()
1651 			(dev->caps.num_uars << PAGE_SHIFT);  in map_bf_area()
1652 	bf_len = pci_resource_len(dev->persist->pdev, 2) -  in map_bf_area()
1653 			(dev->caps.num_uars << PAGE_SHIFT);  in map_bf_area()
1661 static void unmap_bf_area(struct mlx4_dev *dev)  in unmap_bf_area()  argument
1663 	if (mlx4_priv(dev)->bf_mapping)  in unmap_bf_area()
1664 		io_mapping_free(mlx4_priv(dev)->bf_mapping);  in unmap_bf_area()
1667 cycle_t mlx4_read_clock(struct mlx4_dev *dev)  in mlx4_read_clock()  argument
1672 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_read_clock()
1689 static int map_internal_clock(struct mlx4_dev *dev)  in map_internal_clock()  argument
1691 	struct mlx4_priv *priv = mlx4_priv(dev);  in map_internal_clock()
1694 		ioremap(pci_resource_start(dev->persist->pdev,  in map_internal_clock()
1704 int mlx4_get_internal_clock_params(struct mlx4_dev *dev,  in mlx4_get_internal_clock_params()  argument
1707 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_get_internal_clock_params()
1709 	if (mlx4_is_slave(dev))  in mlx4_get_internal_clock_params()
1723 static void unmap_internal_clock(struct mlx4_dev *dev)  in unmap_internal_clock()  argument
1725 	struct mlx4_priv *priv = mlx4_priv(dev);  in unmap_internal_clock()
1731 static void mlx4_close_hca(struct mlx4_dev *dev)  in mlx4_close_hca()  argument
1733 	unmap_internal_clock(dev);  in mlx4_close_hca()
1734 	unmap_bf_area(dev);  in mlx4_close_hca()
1735 	if (mlx4_is_slave(dev))  in mlx4_close_hca()
1736 		mlx4_slave_exit(dev);  in mlx4_close_hca()
1738 		mlx4_CLOSE_HCA(dev, 0);  in mlx4_close_hca()
1739 		mlx4_free_icms(dev);  in mlx4_close_hca()
1743 static void mlx4_close_fw(struct mlx4_dev *dev)  in mlx4_close_fw()  argument
1745 	if (!mlx4_is_slave(dev)) {  in mlx4_close_fw()
1746 		mlx4_UNMAP_FA(dev);  in mlx4_close_fw()
1747 		mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);  in mlx4_close_fw()
1751 static int mlx4_comm_check_offline(struct mlx4_dev *dev)  in mlx4_comm_check_offline()  argument
1758 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_comm_check_offline()
1775 	mlx4_err(dev, "Communication channel is offline.\n");  in mlx4_comm_check_offline()
1779 static void mlx4_reset_vf_support(struct mlx4_dev *dev)  in mlx4_reset_vf_support()  argument
1783 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_reset_vf_support()
1792 		dev->caps.vf_caps |= MLX4_VF_CAP_FLAG_RESET;  in mlx4_reset_vf_support()
1795 static int mlx4_init_slave(struct mlx4_dev *dev)  in mlx4_init_slave()  argument
1797 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_init_slave()
1804 		mlx4_warn(dev, "PF is not ready - Deferring probe\n");  in mlx4_init_slave()
1810 	if (mlx4_comm_check_offline(dev)) {  in mlx4_init_slave()
1811 		mlx4_err(dev, "PF is not responsive, skipping initialization\n");  in mlx4_init_slave()
1815 	mlx4_reset_vf_support(dev);  in mlx4_init_slave()
1816 	mlx4_warn(dev, "Sending reset\n");  in mlx4_init_slave()
1817 	ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0,  in mlx4_init_slave()
1823 			mlx4_warn(dev, "slave is currently in the middle of FLR - Deferring probe\n");  in mlx4_init_slave()
1837 		mlx4_err(dev, "slave driver version is not supported by the master\n");  in mlx4_init_slave()
1841 	mlx4_warn(dev, "Sending vhcr0\n");  in mlx4_init_slave()
1842 	if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48,  in mlx4_init_slave()
1845 	if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32,  in mlx4_init_slave()
1848 	if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16,  in mlx4_init_slave()
1851 	if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma,  in mlx4_init_slave()
1859 	mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, 0);  in mlx4_init_slave()
1865 static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev)  in mlx4_parav_master_pf_caps()  argument
1869 	for (i = 1; i <= dev->caps.num_ports; i++) {  in mlx4_parav_master_pf_caps()
1870 		if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH)  in mlx4_parav_master_pf_caps()
1871 			dev->caps.gid_table_len[i] =  in mlx4_parav_master_pf_caps()
1872 				mlx4_get_slave_num_gids(dev, 0, i);  in mlx4_parav_master_pf_caps()
1874 			dev->caps.gid_table_len[i] = 1;  in mlx4_parav_master_pf_caps()
1875 		dev->caps.pkey_table_len[i] =  in mlx4_parav_master_pf_caps()
1876 			dev->phys_caps.pkey_phys_table_len[i] - 1;  in mlx4_parav_master_pf_caps()
1918 static void choose_steering_mode(struct mlx4_dev *dev,  in choose_steering_mode()  argument
1923 			if (dev->caps.dmfs_high_steer_mode ==  in choose_steering_mode()
1925 				mlx4_err(dev, "DMFS high rate mode not supported\n");  in choose_steering_mode()
1927 				dev->caps.dmfs_high_steer_mode =  in choose_steering_mode()
1934 	    (!mlx4_is_mfunc(dev) ||  in choose_steering_mode()
1936 	     (dev->persist->num_vfs + 1))) &&  in choose_steering_mode()
1939 		dev->oper_log_mgm_entry_size =  in choose_steering_mode()
1941 		dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED;  in choose_steering_mode()
1942 		dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;  in choose_steering_mode()
1943 		dev->caps.fs_log_max_ucast_qp_range_size =  in choose_steering_mode()
1946 		if (dev->caps.dmfs_high_steer_mode !=  in choose_steering_mode()
1948 			dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DISABLE;  in choose_steering_mode()
1949 		if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER &&  in choose_steering_mode()
1950 		    dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)  in choose_steering_mode()
1951 			dev->caps.steering_mode = MLX4_STEERING_MODE_B0;  in choose_steering_mode()
1953 			dev->caps.steering_mode = MLX4_STEERING_MODE_A0;  in choose_steering_mode()
1955 			if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER ||  in choose_steering_mode()
1956 			    dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)  in choose_steering_mode()
1957 …mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags set to use B0 steering - falling back t…  in choose_steering_mode()
1959 		dev->oper_log_mgm_entry_size =  in choose_steering_mode()
1963 		dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev);  in choose_steering_mode()
1965 …mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, modparam log_num_mgm_entry_size…  in choose_steering_mode()
1966 		 mlx4_steering_mode_str(dev->caps.steering_mode),  in choose_steering_mode()
1967 		 dev->oper_log_mgm_entry_size,  in choose_steering_mode()
1971 static void choose_tunnel_offload_mode(struct mlx4_dev *dev,  in choose_tunnel_offload_mode()  argument
1974 	if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED &&  in choose_tunnel_offload_mode()
1976 		dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_VXLAN;  in choose_tunnel_offload_mode()
1978 		dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_NONE;  in choose_tunnel_offload_mode()
1980 	mlx4_dbg(dev, "Tunneling offload mode is: %s\n",  (dev->caps.tunnel_offload_mode  in choose_tunnel_offload_mode()
1984 static int mlx4_validate_optimized_steering(struct mlx4_dev *dev)  in mlx4_validate_optimized_steering()  argument
1989 	if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)  in mlx4_validate_optimized_steering()
1992 	for (i = 1; i <= dev->caps.num_ports; i++) {  in mlx4_validate_optimized_steering()
1993 		if (mlx4_dev_port(dev, i, &port_cap)) {  in mlx4_validate_optimized_steering()
1994 			mlx4_err(dev,  in mlx4_validate_optimized_steering()
1996 		} else if ((dev->caps.dmfs_high_steer_mode !=  in mlx4_validate_optimized_steering()
1999 			    !!(dev->caps.dmfs_high_steer_mode ==  in mlx4_validate_optimized_steering()
2001 			mlx4_err(dev,  in mlx4_validate_optimized_steering()
2004 					dev->caps.dmfs_high_steer_mode),  in mlx4_validate_optimized_steering()
2013 static int mlx4_init_fw(struct mlx4_dev *dev)  in mlx4_init_fw()  argument
2018 	if (!mlx4_is_slave(dev)) {  in mlx4_init_fw()
2019 		err = mlx4_QUERY_FW(dev);  in mlx4_init_fw()
2022 				mlx4_info(dev, "non-primary physical function, skipping\n");  in mlx4_init_fw()
2024 				mlx4_err(dev, "QUERY_FW command failed, aborting\n");  in mlx4_init_fw()
2028 		err = mlx4_load_fw(dev);  in mlx4_init_fw()
2030 			mlx4_err(dev, "Failed to start FW, aborting\n");  in mlx4_init_fw()
2036 		err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);  in mlx4_init_fw()
2038 			mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");  in mlx4_init_fw()
2044 static int mlx4_init_hca(struct mlx4_dev *dev)  in mlx4_init_hca()  argument
2046 	struct mlx4_priv	  *priv = mlx4_priv(dev);  in mlx4_init_hca()
2055 	if (!mlx4_is_slave(dev)) {  in mlx4_init_hca()
2056 		err = mlx4_dev_cap(dev, &dev_cap);  in mlx4_init_hca()
2058 			mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");  in mlx4_init_hca()
2062 		choose_steering_mode(dev, &dev_cap);  in mlx4_init_hca()
2063 		choose_tunnel_offload_mode(dev, &dev_cap);  in mlx4_init_hca()
2065 		if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC &&  in mlx4_init_hca()
2066 		    mlx4_is_master(dev))  in mlx4_init_hca()
2067 			dev->caps.function_caps |= MLX4_FUNC_CAP_DMFS_A0_STATIC;  in mlx4_init_hca()
2069 		err = mlx4_get_phys_port_id(dev);  in mlx4_init_hca()
2071 			mlx4_err(dev, "Fail to get physical port id\n");  in mlx4_init_hca()
2073 		if (mlx4_is_master(dev))  in mlx4_init_hca()
2074 			mlx4_parav_master_pf_caps(dev);  in mlx4_init_hca()
2077 			mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n");  in mlx4_init_hca()
2082 		if (dev->caps.steering_mode ==  in mlx4_init_hca()
2086 		icm_size = mlx4_make_profile(dev, &profile, &dev_cap,  in mlx4_init_hca()
2093 		dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1;  in mlx4_init_hca()
2095 		init_hca.log_uar_sz = ilog2(dev->caps.num_uars);  in mlx4_init_hca()
2098 		if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||  in mlx4_init_hca()
2099 		    dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN)  in mlx4_init_hca()
2102 		err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);  in mlx4_init_hca()
2106 		err = mlx4_INIT_HCA(dev, &init_hca);  in mlx4_init_hca()
2108 			mlx4_err(dev, "INIT_HCA command failed, aborting\n");  in mlx4_init_hca()
2113 			err = mlx4_query_func(dev, &dev_cap);  in mlx4_init_hca()
2115 				mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n");  in mlx4_init_hca()
2118 				dev->caps.num_eqs = dev_cap.max_eqs;  in mlx4_init_hca()
2119 				dev->caps.reserved_eqs = dev_cap.reserved_eqs;  in mlx4_init_hca()
2120 				dev->caps.reserved_uars = dev_cap.reserved_uars;  in mlx4_init_hca()
2128 		if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) {  in mlx4_init_hca()
2130 			err = mlx4_QUERY_HCA(dev, &init_hca);  in mlx4_init_hca()
2132 				mlx4_err(dev, "QUERY_HCA command failed, disable timestamp\n");  in mlx4_init_hca()
2133 				dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;  in mlx4_init_hca()
2135 				dev->caps.hca_core_clock =  in mlx4_init_hca()
2142 			if (!dev->caps.hca_core_clock) {  in mlx4_init_hca()
2143 				dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;  in mlx4_init_hca()
2144 				mlx4_err(dev,  in mlx4_init_hca()
2146 			} else if (map_internal_clock(dev)) {  in mlx4_init_hca()
2151 				dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;  in mlx4_init_hca()
2152 				mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported\n");  in mlx4_init_hca()
2156 		if (dev->caps.dmfs_high_steer_mode !=  in mlx4_init_hca()
2158 			if (mlx4_validate_optimized_steering(dev))  in mlx4_init_hca()
2159 				mlx4_warn(dev, "Optimized steering validation failed\n");  in mlx4_init_hca()
2161 			if (dev->caps.dmfs_high_steer_mode ==  in mlx4_init_hca()
2163 				dev->caps.dmfs_high_rate_qpn_base =  in mlx4_init_hca()
2164 					dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];  in mlx4_init_hca()
2165 				dev->caps.dmfs_high_rate_qpn_range =  in mlx4_init_hca()
2169 			mlx4_dbg(dev, "DMFS high rate steer mode is: %s\n",  in mlx4_init_hca()
2171 					dev->caps.dmfs_high_steer_mode));  in mlx4_init_hca()
2174 		err = mlx4_init_slave(dev);  in mlx4_init_hca()
2177 				mlx4_err(dev, "Failed to initialize slave\n");  in mlx4_init_hca()
2181 		err = mlx4_slave_cap(dev);  in mlx4_init_hca()
2183 			mlx4_err(dev, "Failed to obtain slave caps\n");  in mlx4_init_hca()
2188 	if (map_bf_area(dev))  in mlx4_init_hca()
2189 		mlx4_dbg(dev, "Failed to map blue flame area\n");  in mlx4_init_hca()
2192 	if (!mlx4_is_slave(dev))  in mlx4_init_hca()
2193 		mlx4_set_port_mask(dev);  in mlx4_init_hca()
2195 	err = mlx4_QUERY_ADAPTER(dev, &adapter);  in mlx4_init_hca()
2197 		mlx4_err(dev, "QUERY_ADAPTER command failed, aborting\n");  in mlx4_init_hca()
2202 	err = mlx4_config_dev_retrieval(dev, &params);  in mlx4_init_hca()
2204 		mlx4_err(dev, "Failed to query CONFIG_DEV parameters\n");  in mlx4_init_hca()
2206 		dev->caps.rx_checksum_flags_port[1] = params.rx_csum_flags_port_1;  in mlx4_init_hca()
2207 		dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2;  in mlx4_init_hca()
2210 	memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id);  in mlx4_init_hca()
2215 	unmap_internal_clock(dev);  in mlx4_init_hca()
2216 	unmap_bf_area(dev);  in mlx4_init_hca()
2218 	if (mlx4_is_slave(dev)) {  in mlx4_init_hca()
2219 		kfree(dev->caps.qp0_qkey);  in mlx4_init_hca()
2220 		kfree(dev->caps.qp0_tunnel);  in mlx4_init_hca()
2221 		kfree(dev->caps.qp0_proxy);  in mlx4_init_hca()
2222 		kfree(dev->caps.qp1_tunnel);  in mlx4_init_hca()
2223 		kfree(dev->caps.qp1_proxy);  in mlx4_init_hca()
2227 	if (mlx4_is_slave(dev))  in mlx4_init_hca()
2228 		mlx4_slave_exit(dev);  in mlx4_init_hca()
2230 		mlx4_CLOSE_HCA(dev, 0);  in mlx4_init_hca()
2233 	if (!mlx4_is_slave(dev))  in mlx4_init_hca()
2234 		mlx4_free_icms(dev);  in mlx4_init_hca()
2239 static int mlx4_init_counters_table(struct mlx4_dev *dev)  in mlx4_init_counters_table()  argument
2241 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_init_counters_table()
2244 	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))  in mlx4_init_counters_table()
2247 	if (!dev->caps.max_counters)  in mlx4_init_counters_table()
2250 	nent_pow2 = roundup_pow_of_two(dev->caps.max_counters);  in mlx4_init_counters_table()
2254 				nent_pow2 - dev->caps.max_counters + 1);  in mlx4_init_counters_table()
2257 static void mlx4_cleanup_counters_table(struct mlx4_dev *dev)  in mlx4_cleanup_counters_table()  argument
2259 	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))  in mlx4_cleanup_counters_table()
2262 	if (!dev->caps.max_counters)  in mlx4_cleanup_counters_table()
2265 	mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap);  in mlx4_cleanup_counters_table()
2268 static void mlx4_cleanup_default_counters(struct mlx4_dev *dev)  in mlx4_cleanup_default_counters()  argument
2270 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_cleanup_default_counters()
2273 	for (port = 0; port < dev->caps.num_ports; port++)  in mlx4_cleanup_default_counters()
2275 			mlx4_counter_free(dev,  priv->def_counter[port]);  in mlx4_cleanup_default_counters()
2278 static int mlx4_allocate_default_counters(struct mlx4_dev *dev)  in mlx4_allocate_default_counters()  argument
2280 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_allocate_default_counters()
2284 	for (port = 0; port < dev->caps.num_ports; port++)  in mlx4_allocate_default_counters()
2287 	for (port = 0; port < dev->caps.num_ports; port++) {  in mlx4_allocate_default_counters()
2288 		err = mlx4_counter_alloc(dev, &idx);  in mlx4_allocate_default_counters()
2295 		} else if (mlx4_is_slave(dev) && err == -EINVAL) {  in mlx4_allocate_default_counters()
2296 			priv->def_counter[port] = MLX4_SINK_COUNTER_INDEX(dev);  in mlx4_allocate_default_counters()
2297 			mlx4_warn(dev, "can't allocate counter from old PF driver, using index %d\n",  in mlx4_allocate_default_counters()
2298 				  MLX4_SINK_COUNTER_INDEX(dev));  in mlx4_allocate_default_counters()
2301 			mlx4_err(dev, "%s: failed to allocate default counter port %d err %d\n",  in mlx4_allocate_default_counters()
2303 			mlx4_cleanup_default_counters(dev);  in mlx4_allocate_default_counters()
2307 		mlx4_dbg(dev, "%s: default counter index %d for port %d\n",  in mlx4_allocate_default_counters()
2314 int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)  in __mlx4_counter_alloc()  argument
2316 	struct mlx4_priv *priv = mlx4_priv(dev);  in __mlx4_counter_alloc()
2318 	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))  in __mlx4_counter_alloc()
2323 		*idx = MLX4_SINK_COUNTER_INDEX(dev);  in __mlx4_counter_alloc()
2330 int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)  in mlx4_counter_alloc()  argument
2335 	if (mlx4_is_mfunc(dev)) {  in mlx4_counter_alloc()
2336 		err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER,  in mlx4_counter_alloc()
2344 	return __mlx4_counter_alloc(dev, idx);  in mlx4_counter_alloc()
2348 static int __mlx4_clear_if_stat(struct mlx4_dev *dev,  in __mlx4_clear_if_stat()  argument
2355 	if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev);  in __mlx4_clear_if_stat()
2359 	err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, if_stat_in_mod, 0,  in __mlx4_clear_if_stat()
2363 	mlx4_free_cmd_mailbox(dev, if_stat_mailbox);  in __mlx4_clear_if_stat()
2367 void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx)  in __mlx4_counter_free()  argument
2369 	if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))  in __mlx4_counter_free()
2372 	if (idx == MLX4_SINK_COUNTER_INDEX(dev))  in __mlx4_counter_free()
2375 	__mlx4_clear_if_stat(dev, idx);  in __mlx4_counter_free()
2377 	mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx, MLX4_USE_RR);  in __mlx4_counter_free()
2381 void mlx4_counter_free(struct mlx4_dev *dev, u32 idx)  in mlx4_counter_free()  argument
2385 	if (mlx4_is_mfunc(dev)) {  in mlx4_counter_free()
2387 		mlx4_cmd(dev, in_param, RES_COUNTER, RES_OP_RESERVE,  in mlx4_counter_free()
2392 	__mlx4_counter_free(dev, idx);  in mlx4_counter_free()
2396 int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port)  in mlx4_get_default_counter_index()  argument
2398 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_get_default_counter_index()
2404 void mlx4_set_admin_guid(struct mlx4_dev *dev, __be64 guid, int entry, int port)  in mlx4_set_admin_guid()  argument
2406 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_set_admin_guid()
2412 __be64 mlx4_get_admin_guid(struct mlx4_dev *dev, int entry, int port)  in mlx4_get_admin_guid()  argument
2414 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_get_admin_guid()
2420 void mlx4_set_random_admin_guid(struct mlx4_dev *dev, int entry, int port)  in mlx4_set_random_admin_guid()  argument
2422 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_set_random_admin_guid()
2435 static int mlx4_setup_hca(struct mlx4_dev *dev)  in mlx4_setup_hca()  argument
2437 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_setup_hca()
2442 	err = mlx4_init_uar_table(dev);  in mlx4_setup_hca()
2444 		mlx4_err(dev, "Failed to initialize user access region table, aborting\n");  in mlx4_setup_hca()
2448 	err = mlx4_uar_alloc(dev, &priv->driver_uar);  in mlx4_setup_hca()
2450 		mlx4_err(dev, "Failed to allocate driver access region, aborting\n");  in mlx4_setup_hca()
2456 		mlx4_err(dev, "Couldn't map kernel access region, aborting\n");  in mlx4_setup_hca()
2461 	err = mlx4_init_pd_table(dev);  in mlx4_setup_hca()
2463 		mlx4_err(dev, "Failed to initialize protection domain table, aborting\n");  in mlx4_setup_hca()
2467 	err = mlx4_init_xrcd_table(dev);  in mlx4_setup_hca()
2469 		mlx4_err(dev, "Failed to initialize reliable connection domain table, aborting\n");  in mlx4_setup_hca()
2473 	err = mlx4_init_mr_table(dev);  in mlx4_setup_hca()
2475 		mlx4_err(dev, "Failed to initialize memory region table, aborting\n");  in mlx4_setup_hca()
2479 	if (!mlx4_is_slave(dev)) {  in mlx4_setup_hca()
2480 		err = mlx4_init_mcg_table(dev);  in mlx4_setup_hca()
2482 			mlx4_err(dev, "Failed to initialize multicast group table, aborting\n");  in mlx4_setup_hca()
2485 		err = mlx4_config_mad_demux(dev);  in mlx4_setup_hca()
2487 			mlx4_err(dev, "Failed in config_mad_demux, aborting\n");  in mlx4_setup_hca()
2492 	err = mlx4_init_eq_table(dev);  in mlx4_setup_hca()
2494 		mlx4_err(dev, "Failed to initialize event queue table, aborting\n");  in mlx4_setup_hca()
2498 	err = mlx4_cmd_use_events(dev);  in mlx4_setup_hca()
2500 		mlx4_err(dev, "Failed to switch to event-driven firmware commands, aborting\n");  in mlx4_setup_hca()
2504 	err = mlx4_NOP(dev);  in mlx4_setup_hca()
2506 		if (dev->flags & MLX4_FLAG_MSI_X) {  in mlx4_setup_hca()
2507 			mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n",  in mlx4_setup_hca()
2509 			mlx4_warn(dev, "Trying again without MSI-X\n");  in mlx4_setup_hca()
2511 			mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n",  in mlx4_setup_hca()
2513 			mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");  in mlx4_setup_hca()
2519 	mlx4_dbg(dev, "NOP command IRQ test passed\n");  in mlx4_setup_hca()
2521 	err = mlx4_init_cq_table(dev);  in mlx4_setup_hca()
2523 		mlx4_err(dev, "Failed to initialize completion queue table, aborting\n");  in mlx4_setup_hca()
2527 	err = mlx4_init_srq_table(dev);  in mlx4_setup_hca()
2529 		mlx4_err(dev, "Failed to initialize shared receive queue table, aborting\n");  in mlx4_setup_hca()
2533 	err = mlx4_init_qp_table(dev);  in mlx4_setup_hca()
2535 		mlx4_err(dev, "Failed to initialize queue pair table, aborting\n");  in mlx4_setup_hca()
2539 	if (!mlx4_is_slave(dev)) {  in mlx4_setup_hca()
2540 		err = mlx4_init_counters_table(dev);  in mlx4_setup_hca()
2542 			mlx4_err(dev, "Failed to initialize counters table, aborting\n");  in mlx4_setup_hca()
2547 	err = mlx4_allocate_default_counters(dev);  in mlx4_setup_hca()
2549 		mlx4_err(dev, "Failed to allocate default counters, aborting\n");  in mlx4_setup_hca()
2553 	if (!mlx4_is_slave(dev)) {  in mlx4_setup_hca()
2554 		for (port = 1; port <= dev->caps.num_ports; port++) {  in mlx4_setup_hca()
2556 			err = mlx4_get_port_ib_caps(dev, port,  in mlx4_setup_hca()
2559 				mlx4_warn(dev, "failed to get port %d default ib capabilities (%d). Continuing with caps = 0\n",  in mlx4_setup_hca()
2561 			dev->caps.ib_port_def_cap[port] = ib_port_default_caps;  in mlx4_setup_hca()
2564 			if (mlx4_is_master(dev)) {  in mlx4_setup_hca()
2566 				for (i = 0; i < dev->num_slaves; i++) {  in mlx4_setup_hca()
2567 					if (i == mlx4_master_func_num(dev))  in mlx4_setup_hca()
2574 			if (mlx4_is_mfunc(dev))  in mlx4_setup_hca()
2575 				dev->caps.port_ib_mtu[port] = IB_MTU_2048;  in mlx4_setup_hca()
2577 				dev->caps.port_ib_mtu[port] = IB_MTU_4096;  in mlx4_setup_hca()
2579 			err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ?  in mlx4_setup_hca()
2580 					    dev->caps.pkey_table_len[port] : -1);  in mlx4_setup_hca()
2582 				mlx4_err(dev, "Failed to set port %d, aborting\n",  in mlx4_setup_hca()
2592 	mlx4_cleanup_default_counters(dev);  in mlx4_setup_hca()
2595 	if (!mlx4_is_slave(dev))  in mlx4_setup_hca()
2596 		mlx4_cleanup_counters_table(dev);  in mlx4_setup_hca()
2599 	mlx4_cleanup_qp_table(dev);  in mlx4_setup_hca()
2602 	mlx4_cleanup_srq_table(dev);  in mlx4_setup_hca()
2605 	mlx4_cleanup_cq_table(dev);  in mlx4_setup_hca()
2608 	mlx4_cmd_use_polling(dev);  in mlx4_setup_hca()
2611 	mlx4_cleanup_eq_table(dev);  in mlx4_setup_hca()
2614 	if (!mlx4_is_slave(dev))  in mlx4_setup_hca()
2615 		mlx4_cleanup_mcg_table(dev);  in mlx4_setup_hca()
2618 	mlx4_cleanup_mr_table(dev);  in mlx4_setup_hca()
2621 	mlx4_cleanup_xrcd_table(dev);  in mlx4_setup_hca()
2624 	mlx4_cleanup_pd_table(dev);  in mlx4_setup_hca()
2630 	mlx4_uar_free(dev, &priv->driver_uar);  in mlx4_setup_hca()
2633 	mlx4_cleanup_uar_table(dev);  in mlx4_setup_hca()
2637 static int mlx4_init_affinity_hint(struct mlx4_dev *dev, int port, int eqn)  in mlx4_init_affinity_hint()  argument
2640 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_init_affinity_hint()
2645 	if (eqn > dev->caps.num_comp_vectors)  in mlx4_init_affinity_hint()
2649 		off += mlx4_get_eqs_per_port(dev, i);  in mlx4_init_affinity_hint()
2667 static void mlx4_enable_msi_x(struct mlx4_dev *dev)  in mlx4_enable_msi_x()  argument
2669 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_enable_msi_x()
2675 		int nreq = dev->caps.num_ports * num_online_cpus() + 1;  in mlx4_enable_msi_x()
2677 		nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs,  in mlx4_enable_msi_x()
2689 		nreq = pci_enable_msix_range(dev->persist->pdev, entries, 2,  in mlx4_enable_msi_x()
2697 		dev->caps.num_comp_vectors = nreq - 1;  in mlx4_enable_msi_x()
2701 			    dev->caps.num_ports);  in mlx4_enable_msi_x()
2703 		for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) {  in mlx4_enable_msi_x()
2710 			if (MLX4_IS_LEGACY_EQ_MODE(dev->caps)) {  in mlx4_enable_msi_x()
2712 					    dev->caps.num_ports);  in mlx4_enable_msi_x()
2719 				if (mlx4_init_affinity_hint(dev, port + 1, i))  in mlx4_enable_msi_x()
2720 					mlx4_warn(dev, "Couldn't init hint cpumask for EQ %d\n",  in mlx4_enable_msi_x()
2732 			if ((dev->caps.num_comp_vectors > dev->caps.num_ports) &&  in mlx4_enable_msi_x()
2734 			     (dev->caps.num_comp_vectors / dev->caps.num_ports)) ==  in mlx4_enable_msi_x()
2742 		dev->flags |= MLX4_FLAG_MSI_X;  in mlx4_enable_msi_x()
2749 	dev->caps.num_comp_vectors = 1;  in mlx4_enable_msi_x()
2753 		priv->eq_table.eq[i].irq = dev->persist->pdev->irq;  in mlx4_enable_msi_x()
2756 				    dev->caps.num_ports);  in mlx4_enable_msi_x()
2761 static int mlx4_init_port_info(struct mlx4_dev *dev, int port)  in mlx4_init_port_info()  argument
2763 	struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];  in mlx4_init_port_info()
2766 	info->dev = dev;  in mlx4_init_port_info()
2768 	if (!mlx4_is_slave(dev)) {  in mlx4_init_port_info()
2769 		mlx4_init_mac_table(dev, &info->mac_table);  in mlx4_init_port_info()
2770 		mlx4_init_vlan_table(dev, &info->vlan_table);  in mlx4_init_port_info()
2771 		mlx4_init_roce_gid_table(dev, &info->gid_table);  in mlx4_init_port_info()
2772 		info->base_qpn = mlx4_get_base_qpn(dev, port);  in mlx4_init_port_info()
2777 	if (mlx4_is_mfunc(dev))  in mlx4_init_port_info()
2786 	err = device_create_file(&dev->persist->pdev->dev, &info->port_attr);  in mlx4_init_port_info()
2788 		mlx4_err(dev, "Failed to create file for port %d\n", port);  in mlx4_init_port_info()
2794 	if (mlx4_is_mfunc(dev))  in mlx4_init_port_info()
2803 	err = device_create_file(&dev->persist->pdev->dev,  in mlx4_init_port_info()
2806 		mlx4_err(dev, "Failed to create mtu file for port %d\n", port);  in mlx4_init_port_info()
2807 		device_remove_file(&info->dev->persist->pdev->dev,  in mlx4_init_port_info()
2820 	device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr);  in mlx4_cleanup_port_info()
2821 	device_remove_file(&info->dev->persist->pdev->dev,  in mlx4_cleanup_port_info()
2829 static int mlx4_init_steering(struct mlx4_dev *dev)  in mlx4_init_steering()  argument
2831 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_init_steering()
2832 	int num_entries = dev->caps.num_ports;  in mlx4_init_steering()
2847 static void mlx4_clear_steering(struct mlx4_dev *dev)  in mlx4_clear_steering()  argument
2849 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_clear_steering()
2852 	int num_entries = dev->caps.num_ports;  in mlx4_clear_steering()
2888 static int mlx4_get_ownership(struct mlx4_dev *dev)  in mlx4_get_ownership()  argument
2893 	if (pci_channel_offline(dev->persist->pdev))  in mlx4_get_ownership()
2896 	owner = ioremap(pci_resource_start(dev->persist->pdev, 0) +  in mlx4_get_ownership()
2900 		mlx4_err(dev, "Failed to obtain ownership bit\n");  in mlx4_get_ownership()
2909 static void mlx4_free_ownership(struct mlx4_dev *dev)  in mlx4_free_ownership()  argument
2913 	if (pci_channel_offline(dev->persist->pdev))  in mlx4_free_ownership()
2916 	owner = ioremap(pci_resource_start(dev->persist->pdev, 0) +  in mlx4_free_ownership()
2920 		mlx4_err(dev, "Failed to obtain ownership bit\n");  in mlx4_free_ownership()
2931 static u64 mlx4_enable_sriov(struct mlx4_dev *dev, struct pci_dev *pdev,  in mlx4_enable_sriov()  argument
2934 	u64 dev_flags = dev->flags;  in mlx4_enable_sriov()
2940 		dev->dev_vfs = kcalloc(total_vfs, sizeof(*dev->dev_vfs),  in mlx4_enable_sriov()
2942 		if (!dev->dev_vfs)  in mlx4_enable_sriov()
2948 	if (dev->flags &  MLX4_FLAG_SRIOV) {  in mlx4_enable_sriov()
2950 …	mlx4_err(dev, "SR-IOV was already enabled, but with num_vfs (%d) different than requested (%d)\n",  in mlx4_enable_sriov()
2956 	dev->dev_vfs = kzalloc(total_vfs * sizeof(*dev->dev_vfs), GFP_KERNEL);  in mlx4_enable_sriov()
2957 	if (NULL == dev->dev_vfs) {  in mlx4_enable_sriov()
2958 		mlx4_err(dev, "Failed to allocate memory for VFs\n");  in mlx4_enable_sriov()
2962 	if (!(dev->flags &  MLX4_FLAG_SRIOV)) {  in mlx4_enable_sriov()
2964 			mlx4_err(dev, "requested vfs (%d) > available vfs (%d). Continuing without SR_IOV\n",  in mlx4_enable_sriov()
2969 		mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", total_vfs);  in mlx4_enable_sriov()
2973 		mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d)\n",  in mlx4_enable_sriov()
2977 		mlx4_warn(dev, "Running in master mode\n");  in mlx4_enable_sriov()
2981 		dev->persist->num_vfs = total_vfs;  in mlx4_enable_sriov()
2988 	dev->persist->num_vfs = 0;  in mlx4_enable_sriov()
2989 	kfree(dev->dev_vfs);  in mlx4_enable_sriov()
2990         dev->dev_vfs = NULL;  in mlx4_enable_sriov()
2998 static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,  in mlx4_check_dev_cap()  argument
3005 		mlx4_err(dev, "Requested %d VFs, but FW does not support more than 64\n",  in mlx4_check_dev_cap()
3016 	struct mlx4_dev *dev;  in mlx4_load_one()  local
3024 	dev = &priv->dev;  in mlx4_load_one()
3038 	dev->rev_id = pdev->revision;  in mlx4_load_one()
3039 	dev->numa_node = dev_to_node(&pdev->dev);  in mlx4_load_one()
3043 		mlx4_warn(dev, "Detected virtual function - running in slave mode\n");  in mlx4_load_one()
3044 		dev->flags |= MLX4_FLAG_SLAVE;  in mlx4_load_one()
3049 		err = mlx4_get_ownership(dev);  in mlx4_load_one()
3054 				mlx4_warn(dev, "Multiple PFs not yet supported - Skipping PF\n");  in mlx4_load_one()
3067 		err = mlx4_reset(dev);  in mlx4_load_one()
3069 			mlx4_err(dev, "Failed to reset HCA, aborting\n");  in mlx4_load_one()
3074 			dev->flags = MLX4_FLAG_MASTER;  in mlx4_load_one()
3077 				dev->flags |= MLX4_FLAG_SRIOV;  in mlx4_load_one()
3078 			dev->persist->num_vfs = total_vfs;  in mlx4_load_one()
3085 	dev->persist->state = MLX4_DEVICE_STATE_UP;  in mlx4_load_one()
3088 	err = mlx4_cmd_init(dev);  in mlx4_load_one()
3090 		mlx4_err(dev, "Failed to init command interface, aborting\n");  in mlx4_load_one()
3097 	if (mlx4_is_mfunc(dev)) {  in mlx4_load_one()
3098 		if (mlx4_is_master(dev)) {  in mlx4_load_one()
3099 			dev->num_slaves = MLX4_MAX_NUM_SLAVES;  in mlx4_load_one()
3102 			dev->num_slaves = 0;  in mlx4_load_one()
3103 			err = mlx4_multi_func_init(dev);  in mlx4_load_one()
3105 				mlx4_err(dev, "Failed to init slave mfunc interface, aborting\n");  in mlx4_load_one()
3111 	err = mlx4_init_fw(dev);  in mlx4_load_one()
3113 		mlx4_err(dev, "Failed to init fw, aborting.\n");  in mlx4_load_one()
3117 	if (mlx4_is_master(dev)) {  in mlx4_load_one()
3127 			err = mlx4_QUERY_DEV_CAP(dev, dev_cap);  in mlx4_load_one()
3129 				mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");  in mlx4_load_one()
3133 			if (mlx4_check_dev_cap(dev, dev_cap, nvfs))  in mlx4_load_one()
3137 				u64 dev_flags = mlx4_enable_sriov(dev, pdev,  in mlx4_load_one()
3142 				mlx4_close_fw(dev);  in mlx4_load_one()
3143 				mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);  in mlx4_load_one()
3144 				dev->flags = dev_flags;  in mlx4_load_one()
3145 				if (!SRIOV_VALID_STATE(dev->flags)) {  in mlx4_load_one()
3146 					mlx4_err(dev, "Invalid SRIOV state\n");  in mlx4_load_one()
3149 				err = mlx4_reset(dev);  in mlx4_load_one()
3151 					mlx4_err(dev, "Failed to reset HCA, aborting.\n");  in mlx4_load_one()
3162 			err = mlx4_QUERY_DEV_CAP(dev, dev_cap);  in mlx4_load_one()
3164 				mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");  in mlx4_load_one()
3168 			if (mlx4_check_dev_cap(dev, dev_cap, nvfs))  in mlx4_load_one()
3173 	err = mlx4_init_hca(dev);  in mlx4_load_one()
3178 			mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);  in mlx4_load_one()
3180 			if (dev->flags & MLX4_FLAG_SRIOV) {  in mlx4_load_one()
3183 				if (mlx4_is_master(dev) && !reset_flow)  in mlx4_load_one()
3185 				dev->flags &= ~MLX4_FLAG_SRIOV;  in mlx4_load_one()
3187 			if (!mlx4_is_slave(dev))  in mlx4_load_one()
3188 				mlx4_free_ownership(dev);  in mlx4_load_one()
3189 			dev->flags |= MLX4_FLAG_SLAVE;  in mlx4_load_one()
3190 			dev->flags &= ~MLX4_FLAG_MASTER;  in mlx4_load_one()
3196 	if (mlx4_is_master(dev) && (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {  in mlx4_load_one()
3197 		u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs,  in mlx4_load_one()
3200 		if ((dev->flags ^ dev_flags) & (MLX4_FLAG_MASTER | MLX4_FLAG_SLAVE)) {  in mlx4_load_one()
3201 			mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_VHCR);  in mlx4_load_one()
3202 			dev->flags = dev_flags;  in mlx4_load_one()
3203 			err = mlx4_cmd_init(dev);  in mlx4_load_one()
3208 				mlx4_err(dev, "Failed to init VHCR command interface, aborting\n");  in mlx4_load_one()
3212 			dev->flags = dev_flags;  in mlx4_load_one()
3215 		if (!SRIOV_VALID_STATE(dev->flags)) {  in mlx4_load_one()
3216 			mlx4_err(dev, "Invalid SRIOV state\n");  in mlx4_load_one()
3225 	if (!mlx4_is_slave(dev))  in mlx4_load_one()
3226 		mlx4_check_pcie_caps(dev);  in mlx4_load_one()
3230 	if (mlx4_is_master(dev)) {  in mlx4_load_one()
3231 		if (dev->caps.num_ports < 2 &&  in mlx4_load_one()
3234 			mlx4_err(dev,  in mlx4_load_one()
3236 				 dev->caps.num_ports);  in mlx4_load_one()
3239 		memcpy(dev->persist->nvfs, nvfs, sizeof(dev->persist->nvfs));  in mlx4_load_one()
3242 		     i < sizeof(dev->persist->nvfs)/  in mlx4_load_one()
3243 		     sizeof(dev->persist->nvfs[0]); i++) {  in mlx4_load_one()
3246 			for (j = 0; j < dev->persist->nvfs[i]; ++sum, ++j) {  in mlx4_load_one()
3247 				dev->dev_vfs[sum].min_port = i < 2 ? i + 1 : 1;  in mlx4_load_one()
3248 				dev->dev_vfs[sum].n_ports = i < 2 ? 1 :  in mlx4_load_one()
3249 					dev->caps.num_ports;  in mlx4_load_one()
3256 		err = mlx4_multi_func_init(dev);  in mlx4_load_one()
3258 			mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n");  in mlx4_load_one()
3263 	err = mlx4_alloc_eq_table(dev);  in mlx4_load_one()
3270 	mlx4_enable_msi_x(dev);  in mlx4_load_one()
3271 	if ((mlx4_is_mfunc(dev)) &&  in mlx4_load_one()
3272 	    !(dev->flags & MLX4_FLAG_MSI_X)) {  in mlx4_load_one()
3274 		mlx4_err(dev, "INTx is not supported in multi-function mode, aborting\n");  in mlx4_load_one()
3278 	if (!mlx4_is_slave(dev)) {  in mlx4_load_one()
3279 		err = mlx4_init_steering(dev);  in mlx4_load_one()
3284 	err = mlx4_setup_hca(dev);  in mlx4_load_one()
3285 	if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) &&  in mlx4_load_one()
3286 	    !mlx4_is_mfunc(dev)) {  in mlx4_load_one()
3287 		dev->flags &= ~MLX4_FLAG_MSI_X;  in mlx4_load_one()
3288 		dev->caps.num_comp_vectors = 1;  in mlx4_load_one()
3290 		err = mlx4_setup_hca(dev);  in mlx4_load_one()
3296 	mlx4_init_quotas(dev);  in mlx4_load_one()
3300 	if (mlx4_is_master(dev)) {  in mlx4_load_one()
3301 		err = mlx4_ARM_COMM_CHANNEL(dev);  in mlx4_load_one()
3303 			mlx4_err(dev, " Failed to arm comm channel eq: %x\n",  in mlx4_load_one()
3309 	for (port = 1; port <= dev->caps.num_ports; port++) {  in mlx4_load_one()
3310 		err = mlx4_init_port_info(dev, port);  in mlx4_load_one()
3318 	err = mlx4_register_device(dev);  in mlx4_load_one()
3322 	mlx4_request_modules(dev);  in mlx4_load_one()
3324 	mlx4_sense_init(dev);  in mlx4_load_one()
3325 	mlx4_start_sense(dev);  in mlx4_load_one()
3329 	if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow)  in mlx4_load_one()
3339 	mlx4_cleanup_default_counters(dev);  in mlx4_load_one()
3340 	if (!mlx4_is_slave(dev))  in mlx4_load_one()
3341 		mlx4_cleanup_counters_table(dev);  in mlx4_load_one()
3342 	mlx4_cleanup_qp_table(dev);  in mlx4_load_one()
3343 	mlx4_cleanup_srq_table(dev);  in mlx4_load_one()
3344 	mlx4_cleanup_cq_table(dev);  in mlx4_load_one()
3345 	mlx4_cmd_use_polling(dev);  in mlx4_load_one()
3346 	mlx4_cleanup_eq_table(dev);  in mlx4_load_one()
3347 	mlx4_cleanup_mcg_table(dev);  in mlx4_load_one()
3348 	mlx4_cleanup_mr_table(dev);  in mlx4_load_one()
3349 	mlx4_cleanup_xrcd_table(dev);  in mlx4_load_one()
3350 	mlx4_cleanup_pd_table(dev);  in mlx4_load_one()
3351 	mlx4_cleanup_uar_table(dev);  in mlx4_load_one()
3354 	if (!mlx4_is_slave(dev))  in mlx4_load_one()
3355 		mlx4_clear_steering(dev);  in mlx4_load_one()
3358 	if (dev->flags & MLX4_FLAG_MSI_X)  in mlx4_load_one()
3362 	mlx4_free_eq_table(dev);  in mlx4_load_one()
3365 	if (mlx4_is_master(dev)) {  in mlx4_load_one()
3366 		mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY);  in mlx4_load_one()
3367 		mlx4_multi_func_cleanup(dev);  in mlx4_load_one()
3370 	if (mlx4_is_slave(dev)) {  in mlx4_load_one()
3371 		kfree(dev->caps.qp0_qkey);  in mlx4_load_one()
3372 		kfree(dev->caps.qp0_tunnel);  in mlx4_load_one()
3373 		kfree(dev->caps.qp0_proxy);  in mlx4_load_one()
3374 		kfree(dev->caps.qp1_tunnel);  in mlx4_load_one()
3375 		kfree(dev->caps.qp1_proxy);  in mlx4_load_one()
3379 	mlx4_close_hca(dev);  in mlx4_load_one()
3382 	mlx4_close_fw(dev);  in mlx4_load_one()
3385 	if (mlx4_is_slave(dev))  in mlx4_load_one()
3386 		mlx4_multi_func_cleanup(dev);  in mlx4_load_one()
3389 	mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);  in mlx4_load_one()
3392 	if (dev->flags & MLX4_FLAG_SRIOV && !existing_vfs) {  in mlx4_load_one()
3394 		dev->flags &= ~MLX4_FLAG_SRIOV;  in mlx4_load_one()
3397 	if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow)  in mlx4_load_one()
3400 	kfree(priv->dev.dev_vfs);  in mlx4_load_one()
3402 	if (!mlx4_is_slave(dev))  in mlx4_load_one()
3403 		mlx4_free_ownership(dev);  in mlx4_load_one()
3424 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");  in __mlx4_init_one()
3436 			dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n");  in __mlx4_init_one()
3445 			dev_err(&pdev->dev, "probe_vf module parameter cannot be negative or greater than num_vfs\n");  in __mlx4_init_one()
3451 		dev_err(&pdev->dev,  in __mlx4_init_one()
3460 			dev_err(&pdev->dev,  in __mlx4_init_one()
3472 …dev_err(&pdev->dev, "Missing DCS, aborting (driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%lx)\…  in __mlx4_init_one()
3478 		dev_err(&pdev->dev, "Missing UAR, aborting\n");  in __mlx4_init_one()
3485 		dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");  in __mlx4_init_one()
3493 		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");  in __mlx4_init_one()
3496 			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");  in __mlx4_init_one()
3502 		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit consistent PCI DMA mask\n");  in __mlx4_init_one()
3505 			dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, aborting\n");  in __mlx4_init_one()
3511 	dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024);  in __mlx4_init_one()
3530 				dev_warn(&pdev->dev, "Skipping virtual function:%d\n",  in __mlx4_init_one()
3538 	err = mlx4_catas_init(&priv->dev);  in __mlx4_init_one()
3549 	mlx4_catas_end(&priv->dev);  in __mlx4_init_one()
3563 	struct mlx4_dev *dev;  in mlx4_init_one()  local
3572 	dev       = &priv->dev;  in mlx4_init_one()
3573 	dev->persist = kzalloc(sizeof(*dev->persist), GFP_KERNEL);  in mlx4_init_one()
3574 	if (!dev->persist) {  in mlx4_init_one()
3578 	dev->persist->pdev = pdev;  in mlx4_init_one()
3579 	dev->persist->dev = dev;  in mlx4_init_one()
3580 	pci_set_drvdata(pdev, dev->persist);  in mlx4_init_one()
3582 	mutex_init(&dev->persist->device_state_mutex);  in mlx4_init_one()
3583 	mutex_init(&dev->persist->interface_state_mutex);  in mlx4_init_one()
3587 		kfree(dev->persist);  in mlx4_init_one()
3596 static void mlx4_clean_dev(struct mlx4_dev *dev)  in mlx4_clean_dev()  argument
3598 	struct mlx4_dev_persistent *persist = dev->persist;  in mlx4_clean_dev()
3599 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_clean_dev()
3600 	unsigned long	flags = (dev->flags & RESET_PERSIST_MASK_FLAGS);  in mlx4_clean_dev()
3603 	priv->dev.persist = persist;  in mlx4_clean_dev()
3604 	priv->dev.flags = flags;  in mlx4_clean_dev()
3610 	struct mlx4_dev  *dev  = persist->dev;  in mlx4_unload_one()  local
3611 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_unload_one()
3619 	for (i = 0; i < dev->caps.num_ports; i++) {  in mlx4_unload_one()
3620 		dev->persist->curr_port_type[i] = dev->caps.port_type[i + 1];  in mlx4_unload_one()
3621 		dev->persist->curr_port_poss_type[i] = dev->caps.  in mlx4_unload_one()
3627 	mlx4_stop_sense(dev);  in mlx4_unload_one()
3628 	mlx4_unregister_device(dev);  in mlx4_unload_one()
3630 	for (p = 1; p <= dev->caps.num_ports; p++) {  in mlx4_unload_one()
3632 		mlx4_CLOSE_PORT(dev, p);  in mlx4_unload_one()
3635 	if (mlx4_is_master(dev))  in mlx4_unload_one()
3636 		mlx4_free_resource_tracker(dev,  in mlx4_unload_one()
3639 	mlx4_cleanup_default_counters(dev);  in mlx4_unload_one()
3640 	if (!mlx4_is_slave(dev))  in mlx4_unload_one()
3641 		mlx4_cleanup_counters_table(dev);  in mlx4_unload_one()
3642 	mlx4_cleanup_qp_table(dev);  in mlx4_unload_one()
3643 	mlx4_cleanup_srq_table(dev);  in mlx4_unload_one()
3644 	mlx4_cleanup_cq_table(dev);  in mlx4_unload_one()
3645 	mlx4_cmd_use_polling(dev);  in mlx4_unload_one()
3646 	mlx4_cleanup_eq_table(dev);  in mlx4_unload_one()
3647 	mlx4_cleanup_mcg_table(dev);  in mlx4_unload_one()
3648 	mlx4_cleanup_mr_table(dev);  in mlx4_unload_one()
3649 	mlx4_cleanup_xrcd_table(dev);  in mlx4_unload_one()
3650 	mlx4_cleanup_pd_table(dev);  in mlx4_unload_one()
3652 	if (mlx4_is_master(dev))  in mlx4_unload_one()
3653 		mlx4_free_resource_tracker(dev,  in mlx4_unload_one()
3657 	mlx4_uar_free(dev, &priv->driver_uar);  in mlx4_unload_one()
3658 	mlx4_cleanup_uar_table(dev);  in mlx4_unload_one()
3659 	if (!mlx4_is_slave(dev))  in mlx4_unload_one()
3660 		mlx4_clear_steering(dev);  in mlx4_unload_one()
3661 	mlx4_free_eq_table(dev);  in mlx4_unload_one()
3662 	if (mlx4_is_master(dev))  in mlx4_unload_one()
3663 		mlx4_multi_func_cleanup(dev);  in mlx4_unload_one()
3664 	mlx4_close_hca(dev);  in mlx4_unload_one()
3665 	mlx4_close_fw(dev);  in mlx4_unload_one()
3666 	if (mlx4_is_slave(dev))  in mlx4_unload_one()
3667 		mlx4_multi_func_cleanup(dev);  in mlx4_unload_one()
3668 	mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);  in mlx4_unload_one()
3670 	if (dev->flags & MLX4_FLAG_MSI_X)  in mlx4_unload_one()
3673 	if (!mlx4_is_slave(dev))  in mlx4_unload_one()
3674 		mlx4_free_ownership(dev);  in mlx4_unload_one()
3676 	kfree(dev->caps.qp0_qkey);  in mlx4_unload_one()
3677 	kfree(dev->caps.qp0_tunnel);  in mlx4_unload_one()
3678 	kfree(dev->caps.qp0_proxy);  in mlx4_unload_one()
3679 	kfree(dev->caps.qp1_tunnel);  in mlx4_unload_one()
3680 	kfree(dev->caps.qp1_proxy);  in mlx4_unload_one()
3681 	kfree(dev->dev_vfs);  in mlx4_unload_one()
3683 	mlx4_clean_dev(dev);  in mlx4_unload_one()
3691 	struct mlx4_dev  *dev  = persist->dev;  in mlx4_remove_one()  local
3692 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_remove_one()
3700 	if (mlx4_is_master(dev) && dev->flags & MLX4_FLAG_SRIOV) {  in mlx4_remove_one()
3701 		active_vfs = mlx4_how_many_lives_vf(dev);  in mlx4_remove_one()
3714 		mlx4_info(dev, "%s: interface is down\n", __func__);  in mlx4_remove_one()
3715 	mlx4_catas_end(dev);  in mlx4_remove_one()
3716 	if (dev->flags & MLX4_FLAG_SRIOV && !active_vfs) {  in mlx4_remove_one()
3717 		mlx4_warn(dev, "Disabling SR-IOV\n");  in mlx4_remove_one()
3723 	kfree(dev->persist);  in mlx4_remove_one()
3728 static int restore_current_port_types(struct mlx4_dev *dev,  in restore_current_port_types()  argument
3732 	struct mlx4_priv *priv = mlx4_priv(dev);  in restore_current_port_types()
3735 	mlx4_stop_sense(dev);  in restore_current_port_types()
3738 	for (i = 0; i < dev->caps.num_ports; i++)  in restore_current_port_types()
3739 		dev->caps.possible_type[i + 1] = poss_types[i];  in restore_current_port_types()
3740 	err = mlx4_change_port_types(dev, types);  in restore_current_port_types()
3741 	mlx4_start_sense(dev);  in restore_current_port_types()
3750 	struct mlx4_dev	 *dev  = persist->dev;  in mlx4_restart_one()  local
3751 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_restart_one()
3756 	total_vfs = dev->persist->num_vfs;  in mlx4_restart_one()
3757 	memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));  in mlx4_restart_one()
3762 		mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n",  in mlx4_restart_one()
3767 	err = restore_current_port_types(dev, dev->persist->curr_port_type,  in mlx4_restart_one()
3768 					 dev->persist->curr_port_poss_type);  in mlx4_restart_one()
3770 		mlx4_err(dev, "could not restore original port types (%d)\n",  in mlx4_restart_one()
3829 	mlx4_err(persist->dev, "mlx4_pci_err_detected was called\n");  in mlx4_pci_err_detected()
3847 	struct mlx4_dev	 *dev  = persist->dev;  in mlx4_pci_slot_reset()  local
3848 	struct mlx4_priv *priv = mlx4_priv(dev);  in mlx4_pci_slot_reset()
3853 	mlx4_err(dev, "mlx4_pci_slot_reset was called\n");  in mlx4_pci_slot_reset()
3856 		mlx4_err(dev, "Can not re-enable device, ret=%d\n", ret);  in mlx4_pci_slot_reset()
3864 	total_vfs = dev->persist->num_vfs;  in mlx4_pci_slot_reset()
3865 	memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));  in mlx4_pci_slot_reset()
3872 			mlx4_err(dev, "%s: mlx4_load_one failed, ret=%d\n",  in mlx4_pci_slot_reset()
3877 		ret = restore_current_port_types(dev, dev->persist->  in mlx4_pci_slot_reset()
3878 						 curr_port_type, dev->persist->  in mlx4_pci_slot_reset()
3881 			mlx4_err(dev, "could not restore original port types (%d)\n", ret);  in mlx4_pci_slot_reset()
3893 	mlx4_info(persist->dev, "mlx4_shutdown was called\n");  in mlx4_shutdown()