276 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_send_nolock()  local
278 	unsigned int nentries_mask = src_ring->nentries_mask;  in ath10k_ce_send_nolock()
279 	unsigned int sw_index = src_ring->sw_index;  in ath10k_ce_send_nolock()
280 	unsigned int write_index = src_ring->write_index;  in ath10k_ce_send_nolock()
295 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,  in ath10k_ce_send_nolock()
297 	sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index);  in ath10k_ce_send_nolock()
312 	src_ring->per_transfer_context[write_index] = per_transfer_context;  in ath10k_ce_send_nolock()
321 	src_ring->write_index = write_index;  in ath10k_ce_send_nolock()
330 	struct ath10k_ce_ring *src_ring = pipe->src_ring;  in __ath10k_ce_send_revert()  local
340 	if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))  in __ath10k_ce_send_revert()
343 	if (WARN_ON_ONCE(src_ring->write_index ==  in __ath10k_ce_send_revert()
347 	src_ring->write_index--;  in __ath10k_ce_send_revert()
348 	src_ring->write_index &= src_ring->nentries_mask;  in __ath10k_ce_send_revert()
350 	src_ring->per_transfer_context[src_ring->write_index] = NULL;  in __ath10k_ce_send_revert()
379 	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,  in ath10k_ce_num_free_src_entries()
380 			      pipe->src_ring->write_index,  in ath10k_ce_num_free_src_entries()
381 			      pipe->src_ring->sw_index - 1);  in ath10k_ce_num_free_src_entries()
586 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_completed_send_next_nolock()  local
589 	unsigned int nentries_mask = src_ring->nentries_mask;  in ath10k_ce_completed_send_next_nolock()
590 	unsigned int sw_index = src_ring->sw_index;  in ath10k_ce_completed_send_next_nolock()
594 	if (src_ring->hw_index == sw_index) {  in ath10k_ce_completed_send_next_nolock()
608 		src_ring->hw_index = read_index;  in ath10k_ce_completed_send_next_nolock()
611 	read_index = src_ring->hw_index;  in ath10k_ce_completed_send_next_nolock()
616 	sbase = src_ring->shadow_base;  in ath10k_ce_completed_send_next_nolock()
627 			src_ring->per_transfer_context[sw_index];  in ath10k_ce_completed_send_next_nolock()
630 	src_ring->per_transfer_context[sw_index] = NULL;  in ath10k_ce_completed_send_next_nolock()
634 	src_ring->sw_index = sw_index;  in ath10k_ce_completed_send_next_nolock()
646 	struct ath10k_ce_ring *src_ring;  in ath10k_ce_cancel_send_next()  local
654 	src_ring = ce_state->src_ring;  in ath10k_ce_cancel_send_next()
656 	if (!src_ring)  in ath10k_ce_cancel_send_next()
664 	nentries_mask = src_ring->nentries_mask;  in ath10k_ce_cancel_send_next()
665 	sw_index = src_ring->sw_index;  in ath10k_ce_cancel_send_next()
666 	write_index = src_ring->write_index;  in ath10k_ce_cancel_send_next()
669 		struct ce_desc *base = src_ring->base_addr_owner_space;  in ath10k_ce_cancel_send_next()
680 				src_ring->per_transfer_context[sw_index];  in ath10k_ce_cancel_send_next()
683 		src_ring->per_transfer_context[sw_index] = NULL;  in ath10k_ce_cancel_send_next()
687 		src_ring->sw_index = sw_index;  in ath10k_ce_cancel_send_next()
834 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;  in ath10k_ce_init_src_ring()  local
839 	memset(src_ring->base_addr_owner_space, 0,  in ath10k_ce_init_src_ring()
842 	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);  in ath10k_ce_init_src_ring()
843 	src_ring->sw_index &= src_ring->nentries_mask;  in ath10k_ce_init_src_ring()
844 	src_ring->hw_index = src_ring->sw_index;  in ath10k_ce_init_src_ring()
846 	src_ring->write_index =  in ath10k_ce_init_src_ring()
848 	src_ring->write_index &= src_ring->nentries_mask;  in ath10k_ce_init_src_ring()
851 					 src_ring->base_addr_ce_space);  in ath10k_ce_init_src_ring()
860 		   ce_id, nentries, src_ring->base_addr_owner_space);  in ath10k_ce_init_src_ring()
903 	struct ath10k_ce_ring *src_ring;  in ath10k_ce_alloc_src_ring()  local
909 	src_ring = kzalloc(sizeof(*src_ring) +  in ath10k_ce_alloc_src_ring()
911 			    sizeof(*src_ring->per_transfer_context)),  in ath10k_ce_alloc_src_ring()
913 	if (src_ring == NULL)  in ath10k_ce_alloc_src_ring()
916 	src_ring->nentries = nentries;  in ath10k_ce_alloc_src_ring()
917 	src_ring->nentries_mask = nentries - 1;  in ath10k_ce_alloc_src_ring()
923 	src_ring->base_addr_owner_space_unaligned =  in ath10k_ce_alloc_src_ring()
928 	if (!src_ring->base_addr_owner_space_unaligned) {  in ath10k_ce_alloc_src_ring()
929 		kfree(src_ring);  in ath10k_ce_alloc_src_ring()
933 	src_ring->base_addr_ce_space_unaligned = base_addr;  in ath10k_ce_alloc_src_ring()
935 	src_ring->base_addr_owner_space = PTR_ALIGN(  in ath10k_ce_alloc_src_ring()
936 			src_ring->base_addr_owner_space_unaligned,  in ath10k_ce_alloc_src_ring()
938 	src_ring->base_addr_ce_space = ALIGN(  in ath10k_ce_alloc_src_ring()
939 			src_ring->base_addr_ce_space_unaligned,  in ath10k_ce_alloc_src_ring()
946 	src_ring->shadow_base_unaligned =  in ath10k_ce_alloc_src_ring()
949 	if (!src_ring->shadow_base_unaligned) {  in ath10k_ce_alloc_src_ring()
953 				  src_ring->base_addr_owner_space,  in ath10k_ce_alloc_src_ring()
954 				  src_ring->base_addr_ce_space);  in ath10k_ce_alloc_src_ring()
955 		kfree(src_ring);  in ath10k_ce_alloc_src_ring()
959 	src_ring->shadow_base = PTR_ALIGN(  in ath10k_ce_alloc_src_ring()
960 			src_ring->shadow_base_unaligned,  in ath10k_ce_alloc_src_ring()
963 	return src_ring;  in ath10k_ce_alloc_src_ring()
1112 		ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr);  in ath10k_ce_alloc_pipe()
1113 		if (IS_ERR(ce_state->src_ring)) {  in ath10k_ce_alloc_pipe()
1114 			ret = PTR_ERR(ce_state->src_ring);  in ath10k_ce_alloc_pipe()
1117 			ce_state->src_ring = NULL;  in ath10k_ce_alloc_pipe()
1142 	if (ce_state->src_ring) {  in ath10k_ce_free_pipe()
1143 		kfree(ce_state->src_ring->shadow_base_unaligned);  in ath10k_ce_free_pipe()
1145 				  (ce_state->src_ring->nentries *  in ath10k_ce_free_pipe()
1148 				  ce_state->src_ring->base_addr_owner_space,  in ath10k_ce_free_pipe()
1149 				  ce_state->src_ring->base_addr_ce_space);  in ath10k_ce_free_pipe()
1150 		kfree(ce_state->src_ring);  in ath10k_ce_free_pipe()
1163 	ce_state->src_ring = NULL;  in ath10k_ce_free_pipe()