55 		hw->aq.asq.tail = I40E_VF_ATQT1;  in i40e_adminq_init_regs()
56 		hw->aq.asq.head = I40E_VF_ATQH1;  in i40e_adminq_init_regs()
57 		hw->aq.asq.len  = I40E_VF_ATQLEN1;  in i40e_adminq_init_regs()
58 		hw->aq.asq.bal  = I40E_VF_ATQBAL1;  in i40e_adminq_init_regs()
59 		hw->aq.asq.bah  = I40E_VF_ATQBAH1;  in i40e_adminq_init_regs()
60 		hw->aq.arq.tail = I40E_VF_ARQT1;  in i40e_adminq_init_regs()
61 		hw->aq.arq.head = I40E_VF_ARQH1;  in i40e_adminq_init_regs()
62 		hw->aq.arq.len  = I40E_VF_ARQLEN1;  in i40e_adminq_init_regs()
63 		hw->aq.arq.bal  = I40E_VF_ARQBAL1;  in i40e_adminq_init_regs()
64 		hw->aq.arq.bah  = I40E_VF_ARQBAH1;  in i40e_adminq_init_regs()
66 		hw->aq.asq.tail = I40E_PF_ATQT;  in i40e_adminq_init_regs()
67 		hw->aq.asq.head = I40E_PF_ATQH;  in i40e_adminq_init_regs()
68 		hw->aq.asq.len  = I40E_PF_ATQLEN;  in i40e_adminq_init_regs()
69 		hw->aq.asq.bal  = I40E_PF_ATQBAL;  in i40e_adminq_init_regs()
70 		hw->aq.asq.bah  = I40E_PF_ATQBAH;  in i40e_adminq_init_regs()
71 		hw->aq.arq.tail = I40E_PF_ARQT;  in i40e_adminq_init_regs()
72 		hw->aq.arq.head = I40E_PF_ARQH;  in i40e_adminq_init_regs()
73 		hw->aq.arq.len  = I40E_PF_ARQLEN;  in i40e_adminq_init_regs()
74 		hw->aq.arq.bal  = I40E_PF_ARQBAL;  in i40e_adminq_init_regs()
75 		hw->aq.arq.bah  = I40E_PF_ARQBAH;  in i40e_adminq_init_regs()
87 	ret_code = i40e_allocate_dma_mem(hw, &hw->aq.asq.desc_buf,  in i40e_alloc_adminq_asq_ring()
89 					 (hw->aq.num_asq_entries *  in i40e_alloc_adminq_asq_ring()
95 	ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf,  in i40e_alloc_adminq_asq_ring()
96 					  (hw->aq.num_asq_entries *  in i40e_alloc_adminq_asq_ring()
99 		i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);  in i40e_alloc_adminq_asq_ring()
114 	ret_code = i40e_allocate_dma_mem(hw, &hw->aq.arq.desc_buf,  in i40e_alloc_adminq_arq_ring()
116 					 (hw->aq.num_arq_entries *  in i40e_alloc_adminq_arq_ring()
132 	i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);  in i40e_free_adminq_asq()
144 	i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf);  in i40e_free_adminq_arq()
163 	ret_code = i40e_allocate_virt_mem(hw, &hw->aq.arq.dma_head,  in i40e_alloc_arq_bufs()
164 		(hw->aq.num_arq_entries * sizeof(struct i40e_dma_mem)));  in i40e_alloc_arq_bufs()
167 	hw->aq.arq.r.arq_bi = (struct i40e_dma_mem *)hw->aq.arq.dma_head.va;  in i40e_alloc_arq_bufs()
170 	for (i = 0; i < hw->aq.num_arq_entries; i++) {  in i40e_alloc_arq_bufs()
171 		bi = &hw->aq.arq.r.arq_bi[i];  in i40e_alloc_arq_bufs()
174 						 hw->aq.arq_buf_size,  in i40e_alloc_arq_bufs()
180 		desc = I40E_ADMINQ_DESC(hw->aq.arq, i);  in i40e_alloc_arq_bufs()
183 		if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF)  in i40e_alloc_arq_bufs()
208 		i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);  in i40e_alloc_arq_bufs()
209 	i40e_free_virt_mem(hw, &hw->aq.arq.dma_head);  in i40e_alloc_arq_bufs()
225 	ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.dma_head,  in i40e_alloc_asq_bufs()
226 		(hw->aq.num_asq_entries * sizeof(struct i40e_dma_mem)));  in i40e_alloc_asq_bufs()
229 	hw->aq.asq.r.asq_bi = (struct i40e_dma_mem *)hw->aq.asq.dma_head.va;  in i40e_alloc_asq_bufs()
232 	for (i = 0; i < hw->aq.num_asq_entries; i++) {  in i40e_alloc_asq_bufs()
233 		bi = &hw->aq.asq.r.asq_bi[i];  in i40e_alloc_asq_bufs()
236 						 hw->aq.asq_buf_size,  in i40e_alloc_asq_bufs()
248 		i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);  in i40e_alloc_asq_bufs()
249 	i40e_free_virt_mem(hw, &hw->aq.asq.dma_head);  in i40e_alloc_asq_bufs()
263 	for (i = 0; i < hw->aq.num_arq_entries; i++)  in i40e_free_arq_bufs()
264 		i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);  in i40e_free_arq_bufs()
267 	i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf);  in i40e_free_arq_bufs()
270 	i40e_free_virt_mem(hw, &hw->aq.arq.dma_head);  in i40e_free_arq_bufs()
282 	for (i = 0; i < hw->aq.num_asq_entries; i++)  in i40e_free_asq_bufs()
283 		if (hw->aq.asq.r.asq_bi[i].pa)  in i40e_free_asq_bufs()
284 			i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);  in i40e_free_asq_bufs()
287 	i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf);  in i40e_free_asq_bufs()
290 	i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);  in i40e_free_asq_bufs()
293 	i40e_free_virt_mem(hw, &hw->aq.asq.dma_head);  in i40e_free_asq_bufs()
308 	wr32(hw, hw->aq.asq.head, 0);  in i40e_config_asq_regs()
309 	wr32(hw, hw->aq.asq.tail, 0);  in i40e_config_asq_regs()
312 	wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |  in i40e_config_asq_regs()
314 	wr32(hw, hw->aq.asq.bal, lower_32_bits(hw->aq.asq.desc_buf.pa));  in i40e_config_asq_regs()
315 	wr32(hw, hw->aq.asq.bah, upper_32_bits(hw->aq.asq.desc_buf.pa));  in i40e_config_asq_regs()
318 	reg = rd32(hw, hw->aq.asq.bal);  in i40e_config_asq_regs()
319 	if (reg != lower_32_bits(hw->aq.asq.desc_buf.pa))  in i40e_config_asq_regs()
337 	wr32(hw, hw->aq.arq.head, 0);  in i40e_config_arq_regs()
338 	wr32(hw, hw->aq.arq.tail, 0);  in i40e_config_arq_regs()
341 	wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |  in i40e_config_arq_regs()
343 	wr32(hw, hw->aq.arq.bal, lower_32_bits(hw->aq.arq.desc_buf.pa));  in i40e_config_arq_regs()
344 	wr32(hw, hw->aq.arq.bah, upper_32_bits(hw->aq.arq.desc_buf.pa));  in i40e_config_arq_regs()
347 	wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1);  in i40e_config_arq_regs()
350 	reg = rd32(hw, hw->aq.arq.bal);  in i40e_config_arq_regs()
351 	if (reg != lower_32_bits(hw->aq.arq.desc_buf.pa))  in i40e_config_arq_regs()
374 	if (hw->aq.asq.count > 0) {  in i40e_init_asq()
381 	if ((hw->aq.num_asq_entries == 0) ||  in i40e_init_asq()
382 	    (hw->aq.asq_buf_size == 0)) {  in i40e_init_asq()
387 	hw->aq.asq.next_to_use = 0;  in i40e_init_asq()
388 	hw->aq.asq.next_to_clean = 0;  in i40e_init_asq()
406 	hw->aq.asq.count = hw->aq.num_asq_entries;  in i40e_init_asq()
433 	if (hw->aq.arq.count > 0) {  in i40e_init_arq()
440 	if ((hw->aq.num_arq_entries == 0) ||  in i40e_init_arq()
441 	    (hw->aq.arq_buf_size == 0)) {  in i40e_init_arq()
446 	hw->aq.arq.next_to_use = 0;  in i40e_init_arq()
447 	hw->aq.arq.next_to_clean = 0;  in i40e_init_arq()
465 	hw->aq.arq.count = hw->aq.num_arq_entries;  in i40e_init_arq()
485 	mutex_lock(&hw->aq.asq_mutex);  in i40e_shutdown_asq()
487 	if (hw->aq.asq.count == 0) {  in i40e_shutdown_asq()
493 	wr32(hw, hw->aq.asq.head, 0);  in i40e_shutdown_asq()
494 	wr32(hw, hw->aq.asq.tail, 0);  in i40e_shutdown_asq()
495 	wr32(hw, hw->aq.asq.len, 0);  in i40e_shutdown_asq()
496 	wr32(hw, hw->aq.asq.bal, 0);  in i40e_shutdown_asq()
497 	wr32(hw, hw->aq.asq.bah, 0);  in i40e_shutdown_asq()
499 	hw->aq.asq.count = 0; /* to indicate uninitialized queue */  in i40e_shutdown_asq()
505 	mutex_unlock(&hw->aq.asq_mutex);  in i40e_shutdown_asq()
519 	mutex_lock(&hw->aq.arq_mutex);  in i40e_shutdown_arq()
521 	if (hw->aq.arq.count == 0) {  in i40e_shutdown_arq()
527 	wr32(hw, hw->aq.arq.head, 0);  in i40e_shutdown_arq()
528 	wr32(hw, hw->aq.arq.tail, 0);  in i40e_shutdown_arq()
529 	wr32(hw, hw->aq.arq.len, 0);  in i40e_shutdown_arq()
530 	wr32(hw, hw->aq.arq.bal, 0);  in i40e_shutdown_arq()
531 	wr32(hw, hw->aq.arq.bah, 0);  in i40e_shutdown_arq()
533 	hw->aq.arq.count = 0; /* to indicate uninitialized queue */  in i40e_shutdown_arq()
539 	mutex_unlock(&hw->aq.arq_mutex);  in i40e_shutdown_arq()
562 	if ((hw->aq.num_arq_entries == 0) ||  in i40e_init_adminq()
563 	    (hw->aq.num_asq_entries == 0) ||  in i40e_init_adminq()
564 	    (hw->aq.arq_buf_size == 0) ||  in i40e_init_adminq()
565 	    (hw->aq.asq_buf_size == 0)) {  in i40e_init_adminq()
574 	hw->aq.asq_cmd_timeout = I40E_ASQ_CMD_TIMEOUT;  in i40e_init_adminq()
592 							&hw->aq.fw_maj_ver,  in i40e_init_adminq()
593 							&hw->aq.fw_min_ver,  in i40e_init_adminq()
594 							&hw->aq.fw_build,  in i40e_init_adminq()
595 							&hw->aq.api_maj_ver,  in i40e_init_adminq()
596 							&hw->aq.api_min_ver,  in i40e_init_adminq()
620 	if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) {  in i40e_init_adminq()
627 	hw->aq.nvm_release_on_done = false;  in i40e_init_adminq()
677 	struct i40e_adminq_ring *asq = &(hw->aq.asq);  in i40e_clean_asq()
685 	while (rd32(hw, hw->aq.asq.head) != ntc) {  in i40e_clean_asq()
687 			   "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head));  in i40e_clean_asq()
721 	return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use;  in i40e_asq_done()
750 	mutex_lock(&hw->aq.asq_mutex);  in i40e_asq_send_command()
752 	if (hw->aq.asq.count == 0) {  in i40e_asq_send_command()
759 	hw->aq.asq_last_status = I40E_AQ_RC_OK;  in i40e_asq_send_command()
761 	val = rd32(hw, hw->aq.asq.head);  in i40e_asq_send_command()
762 	if (val >= hw->aq.num_asq_entries) {  in i40e_asq_send_command()
769 	details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use);  in i40e_asq_send_command()
791 	if (buff_size > hw->aq.asq_buf_size) {  in i40e_asq_send_command()
824 	desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use);  in i40e_asq_send_command()
831 		dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]);  in i40e_asq_send_command()
849 	(hw->aq.asq.next_to_use)++;  in i40e_asq_send_command()
850 	if (hw->aq.asq.next_to_use == hw->aq.asq.count)  in i40e_asq_send_command()
851 		hw->aq.asq.next_to_use = 0;  in i40e_asq_send_command()
853 		wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use);  in i40e_asq_send_command()
869 		} while (total_delay < hw->aq.asq_cmd_timeout);  in i40e_asq_send_command()
892 		hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval;  in i40e_asq_send_command()
913 	mutex_unlock(&hw->aq.asq_mutex);  in i40e_asq_send_command()
948 	u16 ntc = hw->aq.arq.next_to_clean;  in i40e_clean_arq_element()
957 	mutex_lock(&hw->aq.arq_mutex);  in i40e_clean_arq_element()
959 	if (hw->aq.arq.count == 0) {  in i40e_clean_arq_element()
967 	ntu = (rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK);  in i40e_clean_arq_element()
975 	desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc);  in i40e_clean_arq_element()
981 		hw->aq.arq_last_status =  in i40e_clean_arq_element()
986 			   hw->aq.arq_last_status);  in i40e_clean_arq_element()
993 		memcpy(e->msg_buf, hw->aq.arq.r.arq_bi[desc_idx].va,  in i40e_clean_arq_element()
998 		      hw->aq.arq_buf_size);  in i40e_clean_arq_element()
1004 	bi = &hw->aq.arq.r.arq_bi[ntc];  in i40e_clean_arq_element()
1008 	if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF)  in i40e_clean_arq_element()
1015 	wr32(hw, hw->aq.arq.tail, ntc);  in i40e_clean_arq_element()
1018 	if (ntc == hw->aq.num_arq_entries)  in i40e_clean_arq_element()
1020 	hw->aq.arq.next_to_clean = ntc;  in i40e_clean_arq_element()
1021 	hw->aq.arq.next_to_use = ntu;  in i40e_clean_arq_element()
1026 		*pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);  in i40e_clean_arq_element()
1029 	mutex_unlock(&hw->aq.arq_mutex);  in i40e_clean_arq_element()
1032 		if (hw->aq.nvm_release_on_done) {  in i40e_clean_arq_element()
1034 			hw->aq.nvm_release_on_done = false;  in i40e_clean_arq_element()
1057 	hw->aq.asq.next_to_use = 0;  in i40e_resume_aq()
1058 	hw->aq.asq.next_to_clean = 0;  in i40e_resume_aq()
1062 	hw->aq.arq.next_to_use = 0;  in i40e_resume_aq()
1063 	hw->aq.arq.next_to_clean = 0;  in i40e_resume_aq()