96 static inline u16 qed_chain_get_prod_idx(struct qed_chain *p_chain)  in qed_chain_get_prod_idx()  argument
98 	return p_chain->prod_idx;  in qed_chain_get_prod_idx()
101 static inline u16 qed_chain_get_cons_idx(struct qed_chain *p_chain)  in qed_chain_get_cons_idx()  argument
103 	return p_chain->cons_idx;  in qed_chain_get_cons_idx()
106 static inline u16 qed_chain_get_elem_left(struct qed_chain *p_chain)  in qed_chain_get_elem_left()  argument
111 	used = ((u32)0x10000u + (u32)(p_chain->prod_idx)) -  in qed_chain_get_elem_left()
112 		(u32)p_chain->cons_idx;  in qed_chain_get_elem_left()
113 	if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR)  in qed_chain_get_elem_left()
114 		used -= p_chain->prod_idx / p_chain->elem_per_page -  in qed_chain_get_elem_left()
115 			p_chain->cons_idx / p_chain->elem_per_page;  in qed_chain_get_elem_left()
117 	return p_chain->capacity - used;  in qed_chain_get_elem_left()
120 static inline u8 qed_chain_is_full(struct qed_chain *p_chain)  in qed_chain_is_full()  argument
122 	return qed_chain_get_elem_left(p_chain) == p_chain->capacity;  in qed_chain_is_full()
125 static inline u8 qed_chain_is_empty(struct qed_chain *p_chain)  in qed_chain_is_empty()  argument
127 	return qed_chain_get_elem_left(p_chain) == 0;  in qed_chain_is_empty()
131 	struct qed_chain *p_chain)  in qed_chain_get_elem_per_page()  argument
133 	return p_chain->elem_per_page;  in qed_chain_get_elem_per_page()
137 	struct qed_chain *p_chain)  in qed_chain_get_usable_per_page()  argument
139 	return p_chain->usable_per_page;  in qed_chain_get_usable_per_page()
143 	struct qed_chain *p_chain)  in qed_chain_get_unusable_per_page()  argument
145 	return p_chain->elem_unusable;  in qed_chain_get_unusable_per_page()
148 static inline u16 qed_chain_get_size(struct qed_chain *p_chain)  in qed_chain_get_size()  argument
150 	return p_chain->size;  in qed_chain_get_size()
154 qed_chain_get_pbl_phys(struct qed_chain *p_chain)  in qed_chain_get_pbl_phys()  argument
156 	return p_chain->pbl.p_phys_table;  in qed_chain_get_pbl_phys()
170 qed_chain_advance_page(struct qed_chain *p_chain,  in qed_chain_advance_page()  argument
176 	switch (p_chain->mode) {  in qed_chain_advance_page()
181 		*idx_to_inc += p_chain->elem_unusable;  in qed_chain_advance_page()
185 		*p_next_elem = p_chain->p_virt_addr;  in qed_chain_advance_page()
192 		if (++(*page_to_inc) == p_chain->page_cnt) {  in qed_chain_advance_page()
194 			*p_next_elem = p_chain->p_virt_addr;  in qed_chain_advance_page()
222 qed_chain_return_multi_produced(struct qed_chain *p_chain,  in qed_chain_return_multi_produced()  argument
225 	p_chain->cons_idx += num;  in qed_chain_return_multi_produced()
226 	test_ans_skip(p_chain, cons_idx);  in qed_chain_return_multi_produced()
237 static inline void qed_chain_return_produced(struct qed_chain *p_chain)  in qed_chain_return_produced()  argument
239 	p_chain->cons_idx++;  in qed_chain_return_produced()
240 	test_ans_skip(p_chain, cons_idx);  in qed_chain_return_produced()
254 static inline void *qed_chain_produce(struct qed_chain *p_chain)  in qed_chain_produce()  argument
258 	if ((p_chain->prod_idx & p_chain->elem_per_page_mask) ==  in qed_chain_produce()
259 	    p_chain->next_page_mask) {  in qed_chain_produce()
260 		qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,  in qed_chain_produce()
261 				       &p_chain->prod_idx,  in qed_chain_produce()
262 				       &p_chain->pbl.prod_page_idx);  in qed_chain_produce()
265 	ret = p_chain->p_prod_elem;  in qed_chain_produce()
266 	p_chain->prod_idx++;  in qed_chain_produce()
267 	p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) +  in qed_chain_produce()
268 					p_chain->elem_size);  in qed_chain_produce()
283 static inline u16 qed_chain_get_capacity(struct qed_chain *p_chain)  in qed_chain_get_capacity()  argument
285 	return p_chain->capacity;  in qed_chain_get_capacity()
297 qed_chain_recycle_consumed(struct qed_chain *p_chain)  in qed_chain_recycle_consumed()  argument
299 	test_ans_skip(p_chain, prod_idx);  in qed_chain_recycle_consumed()
300 	p_chain->prod_idx++;  in qed_chain_recycle_consumed()
313 static inline void *qed_chain_consume(struct qed_chain *p_chain)  in qed_chain_consume()  argument
317 	if ((p_chain->cons_idx & p_chain->elem_per_page_mask) ==  in qed_chain_consume()
318 	    p_chain->next_page_mask) {  in qed_chain_consume()
319 		qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,  in qed_chain_consume()
320 				       &p_chain->cons_idx,  in qed_chain_consume()
321 				       &p_chain->pbl.cons_page_idx);  in qed_chain_consume()
324 	ret = p_chain->p_cons_elem;  in qed_chain_consume()
325 	p_chain->cons_idx++;  in qed_chain_consume()
326 	p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) +  in qed_chain_consume()
327 					p_chain->elem_size);  in qed_chain_consume()
337 static inline void qed_chain_reset(struct qed_chain *p_chain)  in qed_chain_reset()  argument
341 	p_chain->prod_idx	= 0;  in qed_chain_reset()
342 	p_chain->cons_idx	= 0;  in qed_chain_reset()
343 	p_chain->p_cons_elem	= p_chain->p_virt_addr;  in qed_chain_reset()
344 	p_chain->p_prod_elem	= p_chain->p_virt_addr;  in qed_chain_reset()
346 	if (p_chain->mode == QED_CHAIN_MODE_PBL) {  in qed_chain_reset()
347 		p_chain->pbl.prod_page_idx	= p_chain->page_cnt - 1;  in qed_chain_reset()
348 		p_chain->pbl.cons_page_idx	= p_chain->page_cnt - 1;  in qed_chain_reset()
351 	switch (p_chain->intended_use) {  in qed_chain_reset()
359 		for (i = 0; i < p_chain->capacity; i++)  in qed_chain_reset()
360 			qed_chain_recycle_consumed(p_chain);  in qed_chain_reset()
376 static inline void qed_chain_init(struct qed_chain *p_chain,  in qed_chain_init()  argument
385 	p_chain->p_virt_addr	= p_virt_addr;  in qed_chain_init()
386 	p_chain->p_phys_addr	= p_phys_addr;  in qed_chain_init()
387 	p_chain->elem_size	= elem_size;  in qed_chain_init()
388 	p_chain->page_cnt	= page_cnt;  in qed_chain_init()
389 	p_chain->mode		= mode;  in qed_chain_init()
391 	p_chain->intended_use		= intended_use;  in qed_chain_init()
392 	p_chain->elem_per_page		= ELEMS_PER_PAGE(elem_size);  in qed_chain_init()
393 	p_chain->usable_per_page =  in qed_chain_init()
395 	p_chain->capacity		= p_chain->usable_per_page * page_cnt;  in qed_chain_init()
396 	p_chain->size			= p_chain->elem_per_page * page_cnt;  in qed_chain_init()
397 	p_chain->elem_per_page_mask	= p_chain->elem_per_page - 1;  in qed_chain_init()
399 	p_chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(elem_size, mode);  in qed_chain_init()
401 	p_chain->next_page_mask = (p_chain->usable_per_page &  in qed_chain_init()
402 				   p_chain->elem_per_page_mask);  in qed_chain_init()
415 							   p_chain->  in qed_chain_init()
432 						   p_chain->usable_per_page);  in qed_chain_init()
434 		p_next->next_phys.lo	= DMA_LO_LE(p_chain->p_phys_addr);  in qed_chain_init()
435 		p_next->next_phys.hi	= DMA_HI_LE(p_chain->p_phys_addr);  in qed_chain_init()
436 		p_next->next_virt	= p_chain->p_virt_addr;  in qed_chain_init()
438 	qed_chain_reset(p_chain);  in qed_chain_init()
456 qed_chain_pbl_init(struct qed_chain *p_chain,  in qed_chain_pbl_init()  argument
468 	qed_chain_init(p_chain, p_virt_addr, p_phys_addr, page_cnt,  in qed_chain_pbl_init()
471 	p_chain->pbl.p_phys_table = p_phys_pbl;  in qed_chain_pbl_init()
472 	p_chain->pbl.p_virt_table = p_virt_pbl;  in qed_chain_pbl_init()
489 static inline void qed_chain_set_prod(struct qed_chain *p_chain,  in qed_chain_set_prod()  argument
493 	p_chain->prod_idx	= prod_idx;  in qed_chain_set_prod()
494 	p_chain->p_prod_elem	= p_prod_elem;  in qed_chain_set_prod()
507 static inline void *qed_chain_sge_get_elem(struct qed_chain *p_chain,  in qed_chain_sge_get_elem()  argument
512 	if (idx >= p_chain->size)  in qed_chain_sge_get_elem()
515 	ret = (u8 *)p_chain->p_virt_addr + p_chain->elem_size * idx;  in qed_chain_sge_get_elem()
533 qed_chain_sge_inc_cons_prod(struct qed_chain *p_chain,  in qed_chain_sge_inc_cons_prod()  argument
536 	p_chain->prod_idx += cnt;  in qed_chain_sge_inc_cons_prod()
537 	p_chain->cons_idx += cnt;  in qed_chain_sge_inc_cons_prod()