483 static void *__retrieve_octeon_config_info(struct octeon_device *oct,  in __retrieve_octeon_config_info()  argument
486 	u32 oct_id = oct->octeon_id;  in __retrieve_octeon_config_info()
491 		if (oct->chip_id == OCTEON_CN66XX) {  in __retrieve_octeon_config_info()
493 		} else if ((oct->chip_id == OCTEON_CN68XX) &&  in __retrieve_octeon_config_info()
496 		} else if ((oct->chip_id == OCTEON_CN68XX) &&  in __retrieve_octeon_config_info()
507 static int __verify_octeon_config_info(struct octeon_device *oct, void *conf)  in __verify_octeon_config_info()  argument
509 	switch (oct->chip_id) {  in __verify_octeon_config_info()
512 		return lio_validate_cn6xxx_config_info(oct, conf);  in __verify_octeon_config_info()
521 void *oct_get_config_info(struct octeon_device *oct, u16 card_type)  in oct_get_config_info()  argument
525 	conf = __retrieve_octeon_config_info(oct, card_type);  in oct_get_config_info()
529 	if (__verify_octeon_config_info(oct, conf)) {  in oct_get_config_info()
530 		dev_err(&oct->pci_dev->dev, "Configuration verification failed\n");  in oct_get_config_info()
553 int octeon_download_firmware(struct octeon_device *oct, const u8 *data,  in octeon_download_firmware()  argument
566 		dev_err(&oct->pci_dev->dev, "Firmware file too small (%d < %d).\n",  in octeon_download_firmware()
575 		dev_err(&oct->pci_dev->dev, "Unrecognized firmware file.\n");  in octeon_download_firmware()
584 		dev_err(&oct->pci_dev->dev, "Firmware CRC mismatch (0x%08x != 0x%08x).\n",  in octeon_download_firmware()
590 		dev_err(&oct->pci_dev->dev, "Unmatched firmware version. Expected %s, got %s.\n",  in octeon_download_firmware()
596 		dev_err(&oct->pci_dev->dev, "Too many images in firmware file (%d).\n",  in octeon_download_firmware()
601 	dev_info(&oct->pci_dev->dev, "Firmware version: %s\n", h->version);  in octeon_download_firmware()
602 	snprintf(oct->fw_info.liquidio_firmware_version, 32, "LIQUIDIO: %s",  in octeon_download_firmware()
621 			dev_err(&oct->pci_dev->dev,  in octeon_download_firmware()
630 		octeon_pci_write_core_mem(oct, load_addr, p, image_len);  in octeon_download_firmware()
633 		dev_dbg(&oct->pci_dev->dev,  in octeon_download_firmware()
639 	ret = octeon_console_send_cmd(oct, h->bootcmd, 50);  in octeon_download_firmware()
647 void octeon_free_device_mem(struct octeon_device *oct)  in octeon_free_device_mem()  argument
653 		vfree(oct->droq[i]);  in octeon_free_device_mem()
658 		vfree(oct->instr_queue[i]);  in octeon_free_device_mem()
661 	i = oct->octeon_id;  in octeon_free_device_mem()
662 	vfree(oct);  in octeon_free_device_mem()
671 	struct octeon_device *oct;  in octeon_allocate_device_mem()  local
707 	oct = (struct octeon_device *)buf;  in octeon_allocate_device_mem()
708 	oct->priv = (void *)(buf + octdevsize);  in octeon_allocate_device_mem()
709 	oct->chip = (void *)(buf + octdevsize + priv_size);  in octeon_allocate_device_mem()
710 	oct->dispatch.dlist = (struct octeon_dispatch *)  in octeon_allocate_device_mem()
713 	return oct;  in octeon_allocate_device_mem()
720 	struct octeon_device *oct = NULL;  in octeon_allocate_device()  local
729 	oct = octeon_allocate_device_mem(pci_id, priv_size);  in octeon_allocate_device()
730 	if (!oct)  in octeon_allocate_device()
733 	spin_lock_init(&oct->pci_win_lock);  in octeon_allocate_device()
734 	spin_lock_init(&oct->mem_access_lock);  in octeon_allocate_device()
737 	octeon_device[oct_idx] = oct;  in octeon_allocate_device()
739 	oct->octeon_id = oct_idx;  in octeon_allocate_device()
740 	snprintf((oct->device_name), sizeof(oct->device_name),  in octeon_allocate_device()
741 		 "LiquidIO%d", (oct->octeon_id));  in octeon_allocate_device()
743 	return oct;  in octeon_allocate_device()
746 int octeon_setup_instr_queues(struct octeon_device *oct)  in octeon_setup_instr_queues()  argument
752 	if (OCTEON_CN6XXX(oct)) {  in octeon_setup_instr_queues()
755 			CFG_GET_NUM_DEF_TX_DESCS(CHIP_FIELD(oct, cn6xxx, conf));  in octeon_setup_instr_queues()
758 	oct->num_iqs = 0;  in octeon_setup_instr_queues()
761 		oct->instr_queue[i] =  in octeon_setup_instr_queues()
763 		if (!oct->instr_queue[i])  in octeon_setup_instr_queues()
766 		memset(oct->instr_queue[i], 0,  in octeon_setup_instr_queues()
769 		oct->instr_queue[i]->app_ctx = (void *)(size_t)i;  in octeon_setup_instr_queues()
770 		if (octeon_init_instr_queue(oct, i, num_descs))  in octeon_setup_instr_queues()
773 		oct->num_iqs++;  in octeon_setup_instr_queues()
779 int octeon_setup_output_queues(struct octeon_device *oct)  in octeon_setup_output_queues()  argument
786 	if (OCTEON_CN6XXX(oct)) {  in octeon_setup_output_queues()
790 			CFG_GET_NUM_DEF_RX_DESCS(CHIP_FIELD(oct, cn6xxx, conf));  in octeon_setup_output_queues()
792 			CFG_GET_DEF_RX_BUF_SIZE(CHIP_FIELD(oct, cn6xxx, conf));  in octeon_setup_output_queues()
795 	oct->num_oqs = 0;  in octeon_setup_output_queues()
798 		oct->droq[i] = vmalloc(sizeof(*oct->droq[i]));  in octeon_setup_output_queues()
799 		if (!oct->droq[i])  in octeon_setup_output_queues()
802 		memset(oct->droq[i], 0, sizeof(struct octeon_droq));  in octeon_setup_output_queues()
804 		if (octeon_init_droq(oct, i, num_descs, desc_size, NULL))  in octeon_setup_output_queues()
807 		oct->num_oqs++;  in octeon_setup_output_queues()
813 void octeon_set_io_queues_off(struct octeon_device *oct)  in octeon_set_io_queues_off()  argument
817 	octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);  in octeon_set_io_queues_off()
818 	octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);  in octeon_set_io_queues_off()
821 void octeon_set_droq_pkt_op(struct octeon_device *oct,  in octeon_set_droq_pkt_op()  argument
828 	reg_val = octeon_read_csr(oct, CN6XXX_SLI_PKT_OUT_ENB);  in octeon_set_droq_pkt_op()
835 	octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, reg_val);  in octeon_set_droq_pkt_op()
838 int octeon_init_dispatch_list(struct octeon_device *oct)  in octeon_init_dispatch_list()  argument
842 	oct->dispatch.count = 0;  in octeon_init_dispatch_list()
845 		oct->dispatch.dlist[i].opcode = 0;  in octeon_init_dispatch_list()
846 		INIT_LIST_HEAD(&oct->dispatch.dlist[i].list);  in octeon_init_dispatch_list()
850 		octeon_register_reqtype_free_fn(oct, i, NULL);  in octeon_init_dispatch_list()
852 	spin_lock_init(&oct->dispatch.lock);  in octeon_init_dispatch_list()
857 void octeon_delete_dispatch_list(struct octeon_device *oct)  in octeon_delete_dispatch_list()  argument
864 	spin_lock_bh(&oct->dispatch.lock);  in octeon_delete_dispatch_list()
869 		dispatch = &oct->dispatch.dlist[i].list;  in octeon_delete_dispatch_list()
876 		oct->dispatch.dlist[i].opcode = 0;  in octeon_delete_dispatch_list()
879 	oct->dispatch.count = 0;  in octeon_delete_dispatch_list()
881 	spin_unlock_bh(&oct->dispatch.lock);  in octeon_delete_dispatch_list()
948 octeon_register_dispatch_fn(struct octeon_device *oct,  in octeon_register_dispatch_fn()  argument
959 	spin_lock_bh(&oct->dispatch.lock);  in octeon_register_dispatch_fn()
961 	if (oct->dispatch.dlist[idx].opcode == 0) {  in octeon_register_dispatch_fn()
962 		oct->dispatch.dlist[idx].opcode = combined_opcode;  in octeon_register_dispatch_fn()
963 		oct->dispatch.dlist[idx].dispatch_fn = fn;  in octeon_register_dispatch_fn()
964 		oct->dispatch.dlist[idx].arg = fn_arg;  in octeon_register_dispatch_fn()
965 		oct->dispatch.count++;  in octeon_register_dispatch_fn()
966 		spin_unlock_bh(&oct->dispatch.lock);  in octeon_register_dispatch_fn()
970 	spin_unlock_bh(&oct->dispatch.lock);  in octeon_register_dispatch_fn()
975 	pfn = octeon_get_dispatch(oct, opcode, subcode);  in octeon_register_dispatch_fn()
979 		dev_dbg(&oct->pci_dev->dev,  in octeon_register_dispatch_fn()
984 			dev_err(&oct->pci_dev->dev,  in octeon_register_dispatch_fn()
995 		spin_lock_bh(&oct->dispatch.lock);  in octeon_register_dispatch_fn()
996 		list_add(&dispatch->list, &oct->dispatch.dlist[idx].list);  in octeon_register_dispatch_fn()
997 		oct->dispatch.count++;  in octeon_register_dispatch_fn()
998 		spin_unlock_bh(&oct->dispatch.lock);  in octeon_register_dispatch_fn()
1001 		dev_err(&oct->pci_dev->dev,  in octeon_register_dispatch_fn()
1024 octeon_unregister_dispatch_fn(struct octeon_device *oct, u16 opcode,  in octeon_unregister_dispatch_fn()  argument
1034 	spin_lock_bh(&oct->dispatch.lock);  in octeon_unregister_dispatch_fn()
1036 	if (oct->dispatch.count == 0) {  in octeon_unregister_dispatch_fn()
1037 		spin_unlock_bh(&oct->dispatch.lock);  in octeon_unregister_dispatch_fn()
1038 		dev_err(&oct->pci_dev->dev,  in octeon_unregister_dispatch_fn()
1043 	if (oct->dispatch.dlist[idx].opcode == combined_opcode) {  in octeon_unregister_dispatch_fn()
1044 		dispatch = &oct->dispatch.dlist[idx].list;  in octeon_unregister_dispatch_fn()
1047 			oct->dispatch.dlist[idx].opcode =  in octeon_unregister_dispatch_fn()
1049 			oct->dispatch.dlist[idx].dispatch_fn =  in octeon_unregister_dispatch_fn()
1052 			oct->dispatch.dlist[idx].arg =  in octeon_unregister_dispatch_fn()
1057 			oct->dispatch.dlist[idx].opcode = 0;  in octeon_unregister_dispatch_fn()
1058 			oct->dispatch.dlist[idx].dispatch_fn = NULL;  in octeon_unregister_dispatch_fn()
1059 			oct->dispatch.dlist[idx].arg = NULL;  in octeon_unregister_dispatch_fn()
1064 				   &(oct->dispatch.dlist[idx].  in octeon_unregister_dispatch_fn()
1076 		oct->dispatch.count--;  in octeon_unregister_dispatch_fn()
1078 	spin_unlock_bh(&oct->dispatch.lock);  in octeon_unregister_dispatch_fn()
1087 	struct octeon_device *oct = (struct octeon_device *)buf;  in octeon_core_drv_init()  local
1092 	if (OCTEON_CN6XXX(oct))  in octeon_core_drv_init()
1094 			CFG_GET_NUM_NIC_PORTS(CHIP_FIELD(oct, cn6xxx, conf));  in octeon_core_drv_init()
1096 	if (atomic_read(&oct->status) >= OCT_DEV_RUNNING) {  in octeon_core_drv_init()
1097 		dev_err(&oct->pci_dev->dev, "Received CORE OK when device state is 0x%x\n",  in octeon_core_drv_init()
1098 			atomic_read(&oct->status));  in octeon_core_drv_init()
1106 	oct->app_mode = (u32)recv_pkt->rh.r_core_drv_init.app_mode;  in octeon_core_drv_init()
1108 		oct->fw_info.max_nic_ports =  in octeon_core_drv_init()
1110 		oct->fw_info.num_gmx_ports =  in octeon_core_drv_init()
1114 	if (oct->fw_info.max_nic_ports < num_nic_ports) {  in octeon_core_drv_init()
1115 		dev_err(&oct->pci_dev->dev,  in octeon_core_drv_init()
1117 			num_nic_ports, oct->fw_info.max_nic_ports);  in octeon_core_drv_init()
1120 	oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags;  in octeon_core_drv_init()
1121 	oct->fw_info.app_mode = (u32)recv_pkt->rh.r_core_drv_init.app_mode;  in octeon_core_drv_init()
1123 	atomic_set(&oct->status, OCT_DEV_CORE_OK);  in octeon_core_drv_init()
1125 	cs = &core_setup[oct->octeon_id];  in octeon_core_drv_init()
1128 		dev_dbg(&oct->pci_dev->dev, "Core setup bytes expected %u found %d\n",  in octeon_core_drv_init()
1134 	strncpy(oct->boardinfo.name, cs->boardname, OCT_BOARD_NAME);  in octeon_core_drv_init()
1135 	strncpy(oct->boardinfo.serial_number, cs->board_serial_number,  in octeon_core_drv_init()
1140 	oct->boardinfo.major = cs->board_rev_major;  in octeon_core_drv_init()
1141 	oct->boardinfo.minor = cs->board_rev_minor;  in octeon_core_drv_init()
1143 	dev_info(&oct->pci_dev->dev,  in octeon_core_drv_init()
1154 int octeon_get_tx_qsize(struct octeon_device *oct, u32 q_no)  in octeon_get_tx_qsize()  argument
1157 	if (oct && (q_no < MAX_OCTEON_INSTR_QUEUES) &&  in octeon_get_tx_qsize()
1158 	    (oct->io_qmask.iq & (1UL << q_no)))  in octeon_get_tx_qsize()
1159 		return oct->instr_queue[q_no]->max_count;  in octeon_get_tx_qsize()
1164 int octeon_get_rx_qsize(struct octeon_device *oct, u32 q_no)  in octeon_get_rx_qsize()  argument
1166 	if (oct && (q_no < MAX_OCTEON_OUTPUT_QUEUES) &&  in octeon_get_rx_qsize()
1167 	    (oct->io_qmask.oq & (1UL << q_no)))  in octeon_get_rx_qsize()
1168 		return oct->droq[q_no]->max_count;  in octeon_get_rx_qsize()
1173 struct octeon_config *octeon_get_conf(struct octeon_device *oct)  in octeon_get_conf()  argument
1181 	if (OCTEON_CN6XXX(oct)) {  in octeon_get_conf()
1183 			(struct octeon_config *)(CHIP_FIELD(oct, cn6xxx, conf));  in octeon_get_conf()
1205 u64 lio_pci_readq(struct octeon_device *oct, u64 addr)  in lio_pci_readq()  argument
1211 	spin_lock_irqsave(&oct->pci_win_lock, flags);  in lio_pci_readq()
1217 	if ((oct->chip_id == OCTEON_CN66XX) || (oct->chip_id == OCTEON_CN68XX))  in lio_pci_readq()
1219 	writel(addrhi, oct->reg_list.pci_win_rd_addr_hi);  in lio_pci_readq()
1222 	val32 = readl(oct->reg_list.pci_win_rd_addr_hi);  in lio_pci_readq()
1224 	writel(addr & 0xffffffff, oct->reg_list.pci_win_rd_addr_lo);  in lio_pci_readq()
1225 	val32 = readl(oct->reg_list.pci_win_rd_addr_lo);  in lio_pci_readq()
1227 	val64 = readq(oct->reg_list.pci_win_rd_data);  in lio_pci_readq()
1229 	spin_unlock_irqrestore(&oct->pci_win_lock, flags);  in lio_pci_readq()
1234 void lio_pci_writeq(struct octeon_device *oct,  in lio_pci_writeq()  argument
1241 	spin_lock_irqsave(&oct->pci_win_lock, flags);  in lio_pci_writeq()
1243 	writeq(addr, oct->reg_list.pci_win_wr_addr);  in lio_pci_writeq()
1246 	writel(val >> 32, oct->reg_list.pci_win_wr_data_hi);  in lio_pci_writeq()
1248 	val32 = readl(oct->reg_list.pci_win_wr_data_hi);  in lio_pci_writeq()
1250 	writel(val & 0xffffffff, oct->reg_list.pci_win_wr_data_lo);  in lio_pci_writeq()
1252 	spin_unlock_irqrestore(&oct->pci_win_lock, flags);  in lio_pci_writeq()
1255 int octeon_mem_access_ok(struct octeon_device *oct)  in octeon_mem_access_ok()  argument
1260 	u64 lmc0_reset_ctl = lio_pci_readq(oct, CN6XXX_LMC0_RESET_CTL);  in octeon_mem_access_ok()
1267 int octeon_wait_for_ddr_init(struct octeon_device *oct, u32 *timeout)  in octeon_wait_for_ddr_init()  argument
1280 		ret = octeon_mem_access_ok(oct);  in octeon_wait_for_ddr_init()