128 static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring,  in xhci_link_rings()  argument
134 	if (!ring || !first || !last)  in xhci_link_rings()
137 	next = ring->enq_seg->next;  in xhci_link_rings()
138 	xhci_link_segments(xhci, ring->enq_seg, first, ring->type);  in xhci_link_rings()
139 	xhci_link_segments(xhci, last, next, ring->type);  in xhci_link_rings()
140 	ring->num_segs += num_segs;  in xhci_link_rings()
141 	ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs;  in xhci_link_rings()
143 	if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) {  in xhci_link_rings()
144 		ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control  in xhci_link_rings()
148 		ring->last_seg = last;  in xhci_link_rings()
184 		struct xhci_ring *ring,  in xhci_insert_segment_mapping()  argument
200 			key, ring);  in xhci_insert_segment_mapping()
217 		struct xhci_ring *ring,  in xhci_update_stream_segment_mapping()  argument
232 				ring, seg, mem_flags);  in xhci_update_stream_segment_mapping()
255 static void xhci_remove_stream_mapping(struct xhci_ring *ring)  in xhci_remove_stream_mapping()  argument
259 	if (WARN_ON_ONCE(ring->trb_address_map == NULL))  in xhci_remove_stream_mapping()
262 	seg = ring->first_seg;  in xhci_remove_stream_mapping()
264 		xhci_remove_segment_mapping(ring->trb_address_map, seg);  in xhci_remove_stream_mapping()
266 	} while (seg != ring->first_seg);  in xhci_remove_stream_mapping()
269 static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags)  in xhci_update_stream_mapping()  argument
271 	return xhci_update_stream_segment_mapping(ring->trb_address_map, ring,  in xhci_update_stream_mapping()
272 			ring->first_seg, ring->last_seg, mem_flags);  in xhci_update_stream_mapping()
276 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)  in xhci_ring_free()  argument
278 	if (!ring)  in xhci_ring_free()
281 	if (ring->first_seg) {  in xhci_ring_free()
282 		if (ring->type == TYPE_STREAM)  in xhci_ring_free()
283 			xhci_remove_stream_mapping(ring);  in xhci_ring_free()
284 		xhci_free_segments_for_ring(xhci, ring->first_seg);  in xhci_ring_free()
287 	kfree(ring);  in xhci_ring_free()
290 static void xhci_initialize_ring_info(struct xhci_ring *ring,  in xhci_initialize_ring_info()  argument
294 	ring->enqueue = ring->first_seg->trbs;  in xhci_initialize_ring_info()
295 	ring->enq_seg = ring->first_seg;  in xhci_initialize_ring_info()
296 	ring->dequeue = ring->enqueue;  in xhci_initialize_ring_info()
297 	ring->deq_seg = ring->first_seg;  in xhci_initialize_ring_info()
305 	ring->cycle_state = cycle_state;  in xhci_initialize_ring_info()
307 	ring->enq_updates = 0;  in xhci_initialize_ring_info()
308 	ring->deq_updates = 0;  in xhci_initialize_ring_info()
314 	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;  in xhci_initialize_ring_info()
366 	struct xhci_ring	*ring;  in xhci_ring_alloc()  local
369 	ring = kzalloc(sizeof *(ring), flags);  in xhci_ring_alloc()
370 	if (!ring)  in xhci_ring_alloc()
373 	ring->num_segs = num_segs;  in xhci_ring_alloc()
374 	INIT_LIST_HEAD(&ring->td_list);  in xhci_ring_alloc()
375 	ring->type = type;  in xhci_ring_alloc()
377 		return ring;  in xhci_ring_alloc()
379 	ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg,  in xhci_ring_alloc()
380 			&ring->last_seg, num_segs, cycle_state, type, flags);  in xhci_ring_alloc()
387 		ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=  in xhci_ring_alloc()
390 	xhci_initialize_ring_info(ring, cycle_state);  in xhci_ring_alloc()
391 	return ring;  in xhci_ring_alloc()
394 	kfree(ring);  in xhci_ring_alloc()
407 			virt_dev->eps[ep_index].ring;  in xhci_free_or_cache_endpoint_ring()
414 		xhci_ring_free(xhci, virt_dev->eps[ep_index].ring);  in xhci_free_or_cache_endpoint_ring()
419 	virt_dev->eps[ep_index].ring = NULL;  in xhci_free_or_cache_endpoint_ring()
426 			struct xhci_ring *ring, unsigned int cycle_state,  in xhci_reinit_cached_ring()  argument
429 	struct xhci_segment	*seg = ring->first_seg;  in xhci_reinit_cached_ring()
443 	} while (seg != ring->first_seg);  in xhci_reinit_cached_ring()
444 	ring->type = type;  in xhci_reinit_cached_ring()
445 	xhci_initialize_ring_info(ring, cycle_state);  in xhci_reinit_cached_ring()
449 	INIT_LIST_HEAD(&ring->td_list);  in xhci_reinit_cached_ring()
457 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring,  in xhci_ring_expansion()  argument
470 	num_segs = ring->num_segs > num_segs_needed ?  in xhci_ring_expansion()
471 			ring->num_segs : num_segs_needed;  in xhci_ring_expansion()
474 			num_segs, ring->cycle_state, ring->type, flags);  in xhci_ring_expansion()
478 	if (ring->type == TYPE_STREAM)  in xhci_ring_expansion()
479 		ret = xhci_update_stream_segment_mapping(ring->trb_address_map,  in xhci_ring_expansion()
480 						ring, first, last, flags);  in xhci_ring_expansion()
493 	xhci_link_rings(xhci, ring, first, last, num_segs);  in xhci_ring_expansion()
496 			ring->num_segs);  in xhci_ring_expansion()
626 	return ep->ring;  in xhci_dma_to_transfer_ring()
637 		return ep->ring;  in xhci_stream_id_to_ring()
792 	addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue);  in xhci_setup_no_streams_ep_input_ctx()
793 	ep_ctx->deq  = cpu_to_le64(addr | ep->ring->cycle_state);  in xhci_setup_no_streams_ep_input_ctx()
932 		if (dev->eps[i].ring)  in xhci_free_virt_device()
933 			xhci_ring_free(xhci, dev->eps[i].ring);  in xhci_free_virt_device()
1008 	dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, flags);  in xhci_alloc_virt_device()
1009 	if (!dev->eps[0].ring)  in xhci_alloc_virt_device()
1045 	ep_ring = virt_dev->eps[0].ring;  in xhci_copy_ep0_dequeue_into_input_ctx()
1197 	ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma |  in xhci_setup_addressable_virt_dev()
1198 				   dev->eps[0].ring->cycle_state);  in xhci_setup_addressable_virt_dev()