80 static int adf_reserve_ring(struct adf_etr_bank_data *bank, uint32_t ring)  in adf_reserve_ring()  argument
83 	if (bank->ring_mask & (1 << ring)) {  in adf_reserve_ring()
87 	bank->ring_mask |= (1 << ring);  in adf_reserve_ring()
92 static void adf_unreserve_ring(struct adf_etr_bank_data *bank, uint32_t ring)  in adf_unreserve_ring()  argument
95 	bank->ring_mask &= ~(1 << ring);  in adf_unreserve_ring()
99 static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, uint32_t ring)  in adf_enable_ring_irq()  argument
102 	bank->irq_mask |= (1 << ring);  in adf_enable_ring_irq()
109 static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, uint32_t ring)  in adf_disable_ring_irq()  argument
112 	bank->irq_mask &= ~(1 << ring);  in adf_disable_ring_irq()
117 int adf_send_message(struct adf_etr_ring_data *ring, uint32_t *msg)  in adf_send_message()  argument
119 	if (atomic_add_return(1, ring->inflights) >  in adf_send_message()
120 	    ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) {  in adf_send_message()
121 		atomic_dec(ring->inflights);  in adf_send_message()
124 	spin_lock_bh(&ring->lock);  in adf_send_message()
125 	memcpy(ring->base_addr + ring->tail, msg,  in adf_send_message()
126 	       ADF_MSG_SIZE_TO_BYTES(ring->msg_size));  in adf_send_message()
128 	ring->tail = adf_modulo(ring->tail +  in adf_send_message()
129 				ADF_MSG_SIZE_TO_BYTES(ring->msg_size),  in adf_send_message()
130 				ADF_RING_SIZE_MODULO(ring->ring_size));  in adf_send_message()
131 	WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number,  in adf_send_message()
132 			    ring->ring_number, ring->tail);  in adf_send_message()
133 	spin_unlock_bh(&ring->lock);  in adf_send_message()
137 static int adf_handle_response(struct adf_etr_ring_data *ring)  in adf_handle_response()  argument
140 	uint32_t *msg = (uint32_t *)(ring->base_addr + ring->head);  in adf_handle_response()
143 		ring->callback((uint32_t *)msg);  in adf_handle_response()
145 		ring->head = adf_modulo(ring->head +  in adf_handle_response()
146 					ADF_MSG_SIZE_TO_BYTES(ring->msg_size),  in adf_handle_response()
147 					ADF_RING_SIZE_MODULO(ring->ring_size));  in adf_handle_response()
149 		msg = (uint32_t *)(ring->base_addr + ring->head);  in adf_handle_response()
152 		WRITE_CSR_RING_HEAD(ring->bank->csr_addr,  in adf_handle_response()
153 				    ring->bank->bank_number,  in adf_handle_response()
154 				    ring->ring_number, ring->head);  in adf_handle_response()
155 		atomic_sub(msg_counter, ring->inflights);  in adf_handle_response()
160 static void adf_configure_tx_ring(struct adf_etr_ring_data *ring)  in adf_configure_tx_ring()  argument
162 	uint32_t ring_config = BUILD_RING_CONFIG(ring->ring_size);  in adf_configure_tx_ring()
164 	WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number,  in adf_configure_tx_ring()
165 			      ring->ring_number, ring_config);  in adf_configure_tx_ring()
168 static void adf_configure_rx_ring(struct adf_etr_ring_data *ring)  in adf_configure_rx_ring()  argument
171 			BUILD_RESP_RING_CONFIG(ring->ring_size,  in adf_configure_rx_ring()
175 	WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number,  in adf_configure_rx_ring()
176 			      ring->ring_number, ring_config);  in adf_configure_rx_ring()
179 static int adf_init_ring(struct adf_etr_ring_data *ring)  in adf_init_ring()  argument
181 	struct adf_etr_bank_data *bank = ring->bank;  in adf_init_ring()
186 			ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);  in adf_init_ring()
189 	ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev),  in adf_init_ring()
190 					     ring_size_bytes, &ring->dma_addr,  in adf_init_ring()
192 	if (!ring->base_addr)  in adf_init_ring()
195 	memset(ring->base_addr, 0x7F, ring_size_bytes);  in adf_init_ring()
197 	if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {  in adf_init_ring()
200 				  ring->base_addr, ring->dma_addr);  in adf_init_ring()
204 	if (hw_data->tx_rings_mask & (1 << ring->ring_number))  in adf_init_ring()
205 		adf_configure_tx_ring(ring);  in adf_init_ring()
208 		adf_configure_rx_ring(ring);  in adf_init_ring()
210 	ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size);  in adf_init_ring()
211 	WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number,  in adf_init_ring()
212 			    ring->ring_number, ring_base);  in adf_init_ring()
213 	spin_lock_init(&ring->lock);  in adf_init_ring()
217 static void adf_cleanup_ring(struct adf_etr_ring_data *ring)  in adf_cleanup_ring()  argument
220 			ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);  in adf_cleanup_ring()
223 	if (ring->base_addr) {  in adf_cleanup_ring()
224 		memset(ring->base_addr, 0x7F, ring_size_bytes);  in adf_cleanup_ring()
225 		dma_free_coherent(&GET_DEV(ring->bank->accel_dev),  in adf_cleanup_ring()
226 				  ring_size_bytes, ring->base_addr,  in adf_cleanup_ring()
227 				  ring->dma_addr);  in adf_cleanup_ring()
239 	struct adf_etr_ring_data *ring;  in adf_create_ring()  local
278 	ring = &bank->rings[ring_num];  in adf_create_ring()
279 	ring->ring_number = ring_num;  in adf_create_ring()
280 	ring->bank = bank;  in adf_create_ring()
281 	ring->callback = callback;  in adf_create_ring()
282 	ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size);  in adf_create_ring()
283 	ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);  in adf_create_ring()
284 	ring->head = 0;  in adf_create_ring()
285 	ring->tail = 0;  in adf_create_ring()
286 	atomic_set(ring->inflights, 0);  in adf_create_ring()
287 	ret = adf_init_ring(ring);  in adf_create_ring()
292 	adf_update_ring_arb(ring);  in adf_create_ring()
294 	if (adf_ring_debugfs_add(ring, ring_name)) {  in adf_create_ring()
303 		adf_enable_ring_irq(bank, ring->ring_number);  in adf_create_ring()
304 	*ring_ptr = ring;  in adf_create_ring()
307 	adf_cleanup_ring(ring);  in adf_create_ring()
309 	adf_update_ring_arb(ring);  in adf_create_ring()
313 void adf_remove_ring(struct adf_etr_ring_data *ring)  in adf_remove_ring()  argument
315 	struct adf_etr_bank_data *bank = ring->bank;  in adf_remove_ring()
318 	adf_disable_ring_irq(bank, ring->ring_number);  in adf_remove_ring()
322 			      ring->ring_number, 0);  in adf_remove_ring()
324 			    ring->ring_number, 0);  in adf_remove_ring()
325 	adf_ring_debugfs_rm(ring);  in adf_remove_ring()
326 	adf_unreserve_ring(bank, ring->ring_number);  in adf_remove_ring()
328 	adf_update_ring_arb(ring);  in adf_remove_ring()
329 	adf_cleanup_ring(ring);  in adf_remove_ring()
403 	struct adf_etr_ring_data *ring;  in adf_init_bank()  local
426 		ring = &bank->rings[i];  in adf_init_bank()
428 			ring->inflights =  in adf_init_bank()
432 			if (!ring->inflights)  in adf_init_bank()
441 			ring->inflights = tx_ring->inflights;  in adf_init_bank()
454 		ring = &bank->rings[i];  in adf_init_bank()
456 			kfree(ring->inflights);  in adf_init_bank()
535 		struct adf_etr_ring_data *ring = &bank->rings[i];  in cleanup_bank()  local
538 			adf_cleanup_ring(ring);  in cleanup_bank()
541 			kfree(ring->inflights);  in cleanup_bank()