99 static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq)  in qdio_check_ccq()  argument
111 	DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);  in qdio_check_ccq()
126 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,  in qdio_do_eqbs()  argument
129 	int rc, tmp_count = count, tmp_start = start, nr = q->nr, retried = 0;  in qdio_do_eqbs()
132 	qperf_inc(q, eqbs);  in qdio_do_eqbs()
134 	if (!q->is_input_q)  in qdio_do_eqbs()
135 		nr += q->irq_ptr->nr_input_qs;  in qdio_do_eqbs()
137 	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,  in qdio_do_eqbs()
139 	rc = qdio_check_ccq(q, ccq);  in qdio_do_eqbs()
144 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);  in qdio_do_eqbs()
149 		qperf_inc(q, eqbs_partial);  in qdio_do_eqbs()
150 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS part:%02x",  in qdio_do_eqbs()
162 	DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));  in qdio_do_eqbs()
164 	q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE,  in qdio_do_eqbs()
165 		   q->nr, q->first_to_kick, count, q->irq_ptr->int_parm);  in qdio_do_eqbs()
180 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,  in qdio_do_sqbs()  argument
185 	int nr = q->nr;  in qdio_do_sqbs()
190 	qperf_inc(q, sqbs);  in qdio_do_sqbs()
192 	if (!q->is_input_q)  in qdio_do_sqbs()
193 		nr += q->irq_ptr->nr_input_qs;  in qdio_do_sqbs()
195 	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);  in qdio_do_sqbs()
196 	rc = qdio_check_ccq(q, ccq);  in qdio_do_sqbs()
203 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);  in qdio_do_sqbs()
204 		qperf_inc(q, sqbs_partial);  in qdio_do_sqbs()
208 	DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));  in qdio_do_sqbs()
210 	q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE,  in qdio_do_sqbs()
211 		   q->nr, q->first_to_kick, count, q->irq_ptr->int_parm);  in qdio_do_sqbs()
216 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,  in get_buf_states()  argument
223 	if (is_qebsm(q))  in get_buf_states()
224 		return qdio_do_eqbs(q, state, bufnr, count, auto_ack);  in get_buf_states()
228 			__state = q->slsb.val[bufnr];  in get_buf_states()
232 			if ((q->slsb.val[bufnr] & __state) != __state)  in get_buf_states()
234 		} else if (q->slsb.val[bufnr] != __state)  in get_buf_states()
242 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,  in get_buf_state()  argument
245 	return get_buf_states(q, bufnr, state, 1, auto_ack, 0);  in get_buf_state()
249 static inline int set_buf_states(struct qdio_q *q, int bufnr,  in set_buf_states()  argument
254 	if (is_qebsm(q))  in set_buf_states()
255 		return qdio_do_sqbs(q, state, bufnr, count);  in set_buf_states()
258 		xchg(&q->slsb.val[bufnr], state);  in set_buf_states()
264 static inline int set_buf_state(struct qdio_q *q, int bufnr,  in set_buf_state()  argument
267 	return set_buf_states(q, bufnr, state, 1);  in set_buf_state()
273 	struct qdio_q *q;  in qdio_init_buf_states()  local
276 	for_each_input_queue(irq_ptr, q, i)  in qdio_init_buf_states()
277 		set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,  in qdio_init_buf_states()
279 	for_each_output_queue(irq_ptr, q, i)  in qdio_init_buf_states()
280 		set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,  in qdio_init_buf_states()
284 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,  in qdio_siga_sync()  argument
287 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_sync()
291 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);  in qdio_siga_sync()
292 	qperf_inc(q, siga_sync);  in qdio_siga_sync()
294 	if (is_qebsm(q)) {  in qdio_siga_sync()
295 		schid = q->irq_ptr->sch_token;  in qdio_siga_sync()
301 		DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);  in qdio_siga_sync()
305 static inline int qdio_siga_sync_q(struct qdio_q *q)  in qdio_siga_sync_q()  argument
307 	if (q->is_input_q)  in qdio_siga_sync_q()
308 		return qdio_siga_sync(q, 0, q->mask);  in qdio_siga_sync_q()
310 		return qdio_siga_sync(q, q->mask, 0);  in qdio_siga_sync_q()
313 static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit,  in qdio_siga_output()  argument
316 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_output()
322 	if (q->u.out.use_cq && aob != 0) {  in qdio_siga_output()
327 	if (is_qebsm(q)) {  in qdio_siga_output()
328 		schid = q->irq_ptr->sch_token;  in qdio_siga_output()
332 	WARN_ON_ONCE((aob && queue_type(q) != QDIO_IQDIO_QFMT) ||  in qdio_siga_output()
334 	cc = do_siga_output(schid, q->mask, busy_bit, fc, laob);  in qdio_siga_output()
348 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,  in qdio_siga_output()
349 			      "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);  in qdio_siga_output()
350 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);  in qdio_siga_output()
355 static inline int qdio_siga_input(struct qdio_q *q)  in qdio_siga_input()  argument
357 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_input()
361 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);  in qdio_siga_input()
362 	qperf_inc(q, siga_read);  in qdio_siga_input()
364 	if (is_qebsm(q)) {  in qdio_siga_input()
365 		schid = q->irq_ptr->sch_token;  in qdio_siga_input()
369 	cc = do_siga_input(schid, q->mask, fc);  in qdio_siga_input()
371 		DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);  in qdio_siga_input()
375 #define qdio_siga_sync_out(q) qdio_siga_sync(q, ~0U, 0)  argument
376 #define qdio_siga_sync_all(q) qdio_siga_sync(q, ~0U, ~0U)  argument
378 static inline void qdio_sync_queues(struct qdio_q *q)  in qdio_sync_queues()  argument
381 	if (pci_out_supported(q))  in qdio_sync_queues()
382 		qdio_siga_sync_all(q);  in qdio_sync_queues()
384 		qdio_siga_sync_q(q);  in qdio_sync_queues()
387 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,  in debug_get_buf_state()  argument
390 	if (need_siga_sync(q))  in debug_get_buf_state()
391 		qdio_siga_sync_q(q);  in debug_get_buf_state()
392 	return get_buf_states(q, bufnr, state, 1, 0, 0);  in debug_get_buf_state()
395 static inline void qdio_stop_polling(struct qdio_q *q)  in qdio_stop_polling()  argument
397 	if (!q->u.in.polling)  in qdio_stop_polling()
400 	q->u.in.polling = 0;  in qdio_stop_polling()
401 	qperf_inc(q, stop_polling);  in qdio_stop_polling()
404 	if (is_qebsm(q)) {  in qdio_stop_polling()
405 		set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,  in qdio_stop_polling()
406 			       q->u.in.ack_count);  in qdio_stop_polling()
407 		q->u.in.ack_count = 0;  in qdio_stop_polling()
409 		set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);  in qdio_stop_polling()
412 static inline void account_sbals(struct qdio_q *q, unsigned int count)  in account_sbals()  argument
416 	q->q_stats.nr_sbal_total += count;  in account_sbals()
418 		q->q_stats.nr_sbals[7]++;  in account_sbals()
422 	q->q_stats.nr_sbals[pos]++;  in account_sbals()
425 static void process_buffer_error(struct qdio_q *q, int count)  in process_buffer_error()  argument
427 	unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT :  in process_buffer_error()
430 	q->qdio_error = QDIO_ERROR_SLSB_STATE;  in process_buffer_error()
433 	if ((!q->is_input_q &&  in process_buffer_error()
434 	    (q->sbal[q->first_to_check]->element[15].sflags) == 0x10)) {  in process_buffer_error()
435 		qperf_inc(q, target_full);  in process_buffer_error()
436 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",  in process_buffer_error()
437 			      q->first_to_check);  in process_buffer_error()
441 	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));  in process_buffer_error()
442 	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);  in process_buffer_error()
443 	DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);  in process_buffer_error()
445 		  q->sbal[q->first_to_check]->element[14].sflags,  in process_buffer_error()
446 		  q->sbal[q->first_to_check]->element[15].sflags);  in process_buffer_error()
453 	set_buf_states(q, q->first_to_check, state, count);  in process_buffer_error()
456 static inline void inbound_primed(struct qdio_q *q, int count)  in inbound_primed()  argument
460 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count);  in inbound_primed()
463 	if (is_qebsm(q)) {  in inbound_primed()
464 		if (!q->u.in.polling) {  in inbound_primed()
465 			q->u.in.polling = 1;  in inbound_primed()
466 			q->u.in.ack_count = count;  in inbound_primed()
467 			q->u.in.ack_start = q->first_to_check;  in inbound_primed()
472 		set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,  in inbound_primed()
473 			       q->u.in.ack_count);  in inbound_primed()
474 		q->u.in.ack_count = count;  in inbound_primed()
475 		q->u.in.ack_start = q->first_to_check;  in inbound_primed()
483 	new = add_buf(q->first_to_check, count - 1);  in inbound_primed()
484 	if (q->u.in.polling) {  in inbound_primed()
486 		set_buf_state(q, new, SLSB_P_INPUT_ACK);  in inbound_primed()
487 		set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);  in inbound_primed()
489 		q->u.in.polling = 1;  in inbound_primed()
490 		set_buf_state(q, new, SLSB_P_INPUT_ACK);  in inbound_primed()
493 	q->u.in.ack_start = new;  in inbound_primed()
498 	set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);  in inbound_primed()
501 static int get_inbound_buffer_frontier(struct qdio_q *q)  in get_inbound_buffer_frontier()  argument
506 	q->timestamp = get_tod_clock_fast();  in get_inbound_buffer_frontier()
512 	count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);  in get_inbound_buffer_frontier()
513 	stop = add_buf(q->first_to_check, count);  in get_inbound_buffer_frontier()
515 	if (q->first_to_check == stop)  in get_inbound_buffer_frontier()
522 	count = get_buf_states(q, q->first_to_check, &state, count, 1, 0);  in get_inbound_buffer_frontier()
528 		inbound_primed(q, count);  in get_inbound_buffer_frontier()
529 		q->first_to_check = add_buf(q->first_to_check, count);  in get_inbound_buffer_frontier()
530 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)  in get_inbound_buffer_frontier()
531 			qperf_inc(q, inbound_queue_full);  in get_inbound_buffer_frontier()
532 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
533 			account_sbals(q, count);  in get_inbound_buffer_frontier()
536 		process_buffer_error(q, count);  in get_inbound_buffer_frontier()
537 		q->first_to_check = add_buf(q->first_to_check, count);  in get_inbound_buffer_frontier()
538 		atomic_sub(count, &q->nr_buf_used);  in get_inbound_buffer_frontier()
539 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
540 			account_sbals_error(q, count);  in get_inbound_buffer_frontier()
545 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
546 			q->q_stats.nr_sbal_nop++;  in get_inbound_buffer_frontier()
547 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop");  in get_inbound_buffer_frontier()
553 	return q->first_to_check;  in get_inbound_buffer_frontier()
556 static int qdio_inbound_q_moved(struct qdio_q *q)  in qdio_inbound_q_moved()  argument
560 	bufnr = get_inbound_buffer_frontier(q);  in qdio_inbound_q_moved()
562 	if (bufnr != q->last_move) {  in qdio_inbound_q_moved()
563 		q->last_move = bufnr;  in qdio_inbound_q_moved()
564 		if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)  in qdio_inbound_q_moved()
565 			q->u.in.timestamp = get_tod_clock();  in qdio_inbound_q_moved()
571 static inline int qdio_inbound_q_done(struct qdio_q *q)  in qdio_inbound_q_done()  argument
575 	if (!atomic_read(&q->nr_buf_used))  in qdio_inbound_q_done()
578 	if (need_siga_sync(q))  in qdio_inbound_q_done()
579 		qdio_siga_sync_q(q);  in qdio_inbound_q_done()
580 	get_buf_state(q, q->first_to_check, &state, 0);  in qdio_inbound_q_done()
586 	if (is_thinint_irq(q->irq_ptr))  in qdio_inbound_q_done()
597 	if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {  in qdio_inbound_q_done()
598 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",  in qdio_inbound_q_done()
599 			      q->first_to_check);  in qdio_inbound_q_done()
605 static inline int contains_aobs(struct qdio_q *q)  in contains_aobs()  argument
607 	return !q->is_input_q && q->u.out.use_cq;  in contains_aobs()
610 static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count)  in qdio_handle_aobs()  argument
615 	if (!contains_aobs(q))  in qdio_handle_aobs()
619 		get_buf_state(q, b, &state, 0);  in qdio_handle_aobs()
621 			struct qaob *aob = q->u.out.aobs[b];  in qdio_handle_aobs()
625 			q->u.out.sbal_state[b].flags |=  in qdio_handle_aobs()
627 			q->u.out.aobs[b] = NULL;  in qdio_handle_aobs()
629 			q->u.out.sbal_state[b].aob = NULL;  in qdio_handle_aobs()
635 static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q,  in qdio_aob_for_buffer()  argument
640 	if (!q->use_cq)  in qdio_aob_for_buffer()
643 	if (!q->aobs[bufnr]) {  in qdio_aob_for_buffer()
645 		q->aobs[bufnr] = aob;  in qdio_aob_for_buffer()
647 	if (q->aobs[bufnr]) {  in qdio_aob_for_buffer()
648 		q->sbal_state[bufnr].flags = QDIO_OUTBUF_STATE_FLAG_NONE;  in qdio_aob_for_buffer()
649 		q->sbal_state[bufnr].aob = q->aobs[bufnr];  in qdio_aob_for_buffer()
650 		q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user;  in qdio_aob_for_buffer()
651 		phys_aob = virt_to_phys(q->aobs[bufnr]);  in qdio_aob_for_buffer()
659 static void qdio_kick_handler(struct qdio_q *q)  in qdio_kick_handler()  argument
661 	int start = q->first_to_kick;  in qdio_kick_handler()
662 	int end = q->first_to_check;  in qdio_kick_handler()
665 	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))  in qdio_kick_handler()
670 	if (q->is_input_q) {  in qdio_kick_handler()
671 		qperf_inc(q, inbound_handler);  in qdio_kick_handler()
672 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count);  in qdio_kick_handler()
674 		qperf_inc(q, outbound_handler);  in qdio_kick_handler()
675 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x",  in qdio_kick_handler()
679 	qdio_handle_aobs(q, start, count);  in qdio_kick_handler()
681 	q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,  in qdio_kick_handler()
682 		   q->irq_ptr->int_parm);  in qdio_kick_handler()
685 	q->first_to_kick = end;  in qdio_kick_handler()
686 	q->qdio_error = 0;  in qdio_kick_handler()
689 static void __qdio_inbound_processing(struct qdio_q *q)  in __qdio_inbound_processing()  argument
691 	qperf_inc(q, tasklet_inbound);  in __qdio_inbound_processing()
693 	if (!qdio_inbound_q_moved(q))  in __qdio_inbound_processing()
696 	qdio_kick_handler(q);  in __qdio_inbound_processing()
698 	if (!qdio_inbound_q_done(q)) {  in __qdio_inbound_processing()
700 		qperf_inc(q, tasklet_inbound_resched);  in __qdio_inbound_processing()
701 		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {  in __qdio_inbound_processing()
702 			tasklet_schedule(&q->tasklet);  in __qdio_inbound_processing()
707 	qdio_stop_polling(q);  in __qdio_inbound_processing()
712 	if (!qdio_inbound_q_done(q)) {  in __qdio_inbound_processing()
713 		qperf_inc(q, tasklet_inbound_resched2);  in __qdio_inbound_processing()
714 		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))  in __qdio_inbound_processing()
715 			tasklet_schedule(&q->tasklet);  in __qdio_inbound_processing()
721 	struct qdio_q *q = (struct qdio_q *)data;  in qdio_inbound_processing()  local
722 	__qdio_inbound_processing(q);  in qdio_inbound_processing()
725 static int get_outbound_buffer_frontier(struct qdio_q *q)  in get_outbound_buffer_frontier()  argument
730 	q->timestamp = get_tod_clock_fast();  in get_outbound_buffer_frontier()
732 	if (need_siga_sync(q))  in get_outbound_buffer_frontier()
733 		if (((queue_type(q) != QDIO_IQDIO_QFMT) &&  in get_outbound_buffer_frontier()
734 		    !pci_out_supported(q)) ||  in get_outbound_buffer_frontier()
735 		    (queue_type(q) == QDIO_IQDIO_QFMT &&  in get_outbound_buffer_frontier()
736 		    multicast_outbound(q)))  in get_outbound_buffer_frontier()
737 			qdio_siga_sync_q(q);  in get_outbound_buffer_frontier()
743 	count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);  in get_outbound_buffer_frontier()
744 	stop = add_buf(q->first_to_check, count);  in get_outbound_buffer_frontier()
745 	if (q->first_to_check == stop)  in get_outbound_buffer_frontier()
748 	count = get_buf_states(q, q->first_to_check, &state, count, 0, 1);  in get_outbound_buffer_frontier()
755 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,  in get_outbound_buffer_frontier()
756 			"out empty:%1d %02x", q->nr, count);  in get_outbound_buffer_frontier()
758 		atomic_sub(count, &q->nr_buf_used);  in get_outbound_buffer_frontier()
759 		q->first_to_check = add_buf(q->first_to_check, count);  in get_outbound_buffer_frontier()
760 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
761 			account_sbals(q, count);  in get_outbound_buffer_frontier()
765 		process_buffer_error(q, count);  in get_outbound_buffer_frontier()
766 		q->first_to_check = add_buf(q->first_to_check, count);  in get_outbound_buffer_frontier()
767 		atomic_sub(count, &q->nr_buf_used);  in get_outbound_buffer_frontier()
768 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
769 			account_sbals_error(q, count);  in get_outbound_buffer_frontier()
773 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
774 			q->q_stats.nr_sbal_nop++;  in get_outbound_buffer_frontier()
775 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",  in get_outbound_buffer_frontier()
776 			      q->nr);  in get_outbound_buffer_frontier()
786 	return q->first_to_check;  in get_outbound_buffer_frontier()
790 static inline int qdio_outbound_q_done(struct qdio_q *q)  in qdio_outbound_q_done()  argument
792 	return atomic_read(&q->nr_buf_used) == 0;  in qdio_outbound_q_done()
795 static inline int qdio_outbound_q_moved(struct qdio_q *q)  in qdio_outbound_q_moved()  argument
799 	bufnr = get_outbound_buffer_frontier(q);  in qdio_outbound_q_moved()
801 	if (bufnr != q->last_move) {  in qdio_outbound_q_moved()
802 		q->last_move = bufnr;  in qdio_outbound_q_moved()
803 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);  in qdio_outbound_q_moved()
809 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned long aob)  in qdio_kick_outbound_q()  argument
814 	if (!need_siga_out(q))  in qdio_kick_outbound_q()
817 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);  in qdio_kick_outbound_q()
819 	qperf_inc(q, siga_write);  in qdio_kick_outbound_q()
821 	cc = qdio_siga_output(q, &busy_bit, aob);  in qdio_kick_outbound_q()
831 			DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);  in qdio_kick_outbound_q()
834 			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);  in qdio_kick_outbound_q()
840 		DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);  in qdio_kick_outbound_q()
845 		DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);  in qdio_kick_outbound_q()
851 static void __qdio_outbound_processing(struct qdio_q *q)  in __qdio_outbound_processing()  argument
853 	qperf_inc(q, tasklet_outbound);  in __qdio_outbound_processing()
854 	WARN_ON_ONCE(atomic_read(&q->nr_buf_used) < 0);  in __qdio_outbound_processing()
856 	if (qdio_outbound_q_moved(q))  in __qdio_outbound_processing()
857 		qdio_kick_handler(q);  in __qdio_outbound_processing()
859 	if (queue_type(q) == QDIO_ZFCP_QFMT)  in __qdio_outbound_processing()
860 		if (!pci_out_supported(q) && !qdio_outbound_q_done(q))  in __qdio_outbound_processing()
863 	if (q->u.out.pci_out_enabled)  in __qdio_outbound_processing()
871 	if (qdio_outbound_q_done(q))  in __qdio_outbound_processing()
872 		del_timer(&q->u.out.timer);  in __qdio_outbound_processing()
874 		if (!timer_pending(&q->u.out.timer))  in __qdio_outbound_processing()
875 			mod_timer(&q->u.out.timer, jiffies + 10 * HZ);  in __qdio_outbound_processing()
879 	if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))  in __qdio_outbound_processing()
881 	tasklet_schedule(&q->tasklet);  in __qdio_outbound_processing()
887 	struct qdio_q *q = (struct qdio_q *)data;  in qdio_outbound_processing()  local
888 	__qdio_outbound_processing(q);  in qdio_outbound_processing()
893 	struct qdio_q *q = (struct qdio_q *)data;  in qdio_outbound_timer()  local
895 	if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))  in qdio_outbound_timer()
897 	tasklet_schedule(&q->tasklet);  in qdio_outbound_timer()
900 static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)  in qdio_check_outbound_after_thinint()  argument
905 	if (!pci_out_supported(q))  in qdio_check_outbound_after_thinint()
908 	for_each_output_queue(q->irq_ptr, out, i)  in qdio_check_outbound_after_thinint()
913 static void __tiqdio_inbound_processing(struct qdio_q *q)  in __tiqdio_inbound_processing()  argument
915 	qperf_inc(q, tasklet_inbound);  in __tiqdio_inbound_processing()
916 	if (need_siga_sync(q) && need_siga_sync_after_ai(q))  in __tiqdio_inbound_processing()
917 		qdio_sync_queues(q);  in __tiqdio_inbound_processing()
923 	qdio_check_outbound_after_thinint(q);  in __tiqdio_inbound_processing()
925 	if (!qdio_inbound_q_moved(q))  in __tiqdio_inbound_processing()
928 	qdio_kick_handler(q);  in __tiqdio_inbound_processing()
930 	if (!qdio_inbound_q_done(q)) {  in __tiqdio_inbound_processing()
931 		qperf_inc(q, tasklet_inbound_resched);  in __tiqdio_inbound_processing()
932 		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {  in __tiqdio_inbound_processing()
933 			tasklet_schedule(&q->tasklet);  in __tiqdio_inbound_processing()
938 	qdio_stop_polling(q);  in __tiqdio_inbound_processing()
943 	if (!qdio_inbound_q_done(q)) {  in __tiqdio_inbound_processing()
944 		qperf_inc(q, tasklet_inbound_resched2);  in __tiqdio_inbound_processing()
945 		if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))  in __tiqdio_inbound_processing()
946 			tasklet_schedule(&q->tasklet);  in __tiqdio_inbound_processing()
952 	struct qdio_q *q = (struct qdio_q *)data;  in tiqdio_inbound_processing()  local
953 	__tiqdio_inbound_processing(q);  in tiqdio_inbound_processing()
978 	struct qdio_q *q;  in qdio_int_handler_pci()  local
983 	for_each_input_queue(irq_ptr, q, i) {  in qdio_int_handler_pci()
984 		if (q->u.in.queue_start_poll) {  in qdio_int_handler_pci()
987 				     &q->u.in.queue_irq_state)) {  in qdio_int_handler_pci()
988 				qperf_inc(q, int_discarded);  in qdio_int_handler_pci()
991 			q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,  in qdio_int_handler_pci()
992 						 q->irq_ptr->int_parm);  in qdio_int_handler_pci()
994 			tasklet_schedule(&q->tasklet);  in qdio_int_handler_pci()
1001 	for_each_output_queue(irq_ptr, q, i) {  in qdio_int_handler_pci()
1002 		if (qdio_outbound_q_done(q))  in qdio_int_handler_pci()
1004 		if (need_siga_sync(q) && need_siga_sync_out_after_pci(q))  in qdio_int_handler_pci()
1005 			qdio_siga_sync_q(q);  in qdio_int_handler_pci()
1006 		tasklet_schedule(&q->tasklet);  in qdio_int_handler_pci()
1014 	struct qdio_q *q;  in qdio_handle_activate_check()  local
1022 		q = irq_ptr->input_qs[0];  in qdio_handle_activate_check()
1024 		q = irq_ptr->output_qs[0];  in qdio_handle_activate_check()
1030 	count = sub_buf(q->first_to_check, q->first_to_kick);  in qdio_handle_activate_check()
1031 	q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE,  in qdio_handle_activate_check()
1032 		   q->nr, q->first_to_kick, count, irq_ptr->int_parm);  in qdio_handle_activate_check()
1137 	struct qdio_q *q;  in qdio_shutdown_queues()  local
1140 	for_each_input_queue(irq_ptr, q, i)  in qdio_shutdown_queues()
1141 		tasklet_kill(&q->tasklet);  in qdio_shutdown_queues()
1143 	for_each_output_queue(irq_ptr, q, i) {  in qdio_shutdown_queues()
1144 		del_timer(&q->u.out.timer);  in qdio_shutdown_queues()
1145 		tasklet_kill(&q->tasklet);  in qdio_shutdown_queues()
1310 	struct qdio_q *q = irq_ptr->input_qs[0];  in qdio_detect_hsicq()  local
1313 	if (irq_ptr->nr_input_qs > 1 && queue_type(q) == QDIO_IQDIO_QFMT)  in qdio_detect_hsicq()
1316 	for_each_output_queue(irq_ptr, q, i) {  in qdio_detect_hsicq()
1318 			if (qdio_enable_async_operation(&q->u.out) < 0) {  in qdio_detect_hsicq()
1323 			qdio_disable_async_operation(&q->u.out);  in qdio_detect_hsicq()
1495 static int handle_inbound(struct qdio_q *q, unsigned int callflags,  in handle_inbound()  argument
1500 	qperf_inc(q, inbound_call);  in handle_inbound()
1502 	if (!q->u.in.polling)  in handle_inbound()
1508 		q->u.in.polling = 0;  in handle_inbound()
1509 		q->u.in.ack_count = 0;  in handle_inbound()
1511 	} else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {  in handle_inbound()
1512 		if (is_qebsm(q)) {  in handle_inbound()
1515 			diff = sub_buf(diff, q->u.in.ack_start);  in handle_inbound()
1516 			q->u.in.ack_count -= diff;  in handle_inbound()
1517 			if (q->u.in.ack_count <= 0) {  in handle_inbound()
1518 				q->u.in.polling = 0;  in handle_inbound()
1519 				q->u.in.ack_count = 0;  in handle_inbound()
1522 			q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);  in handle_inbound()
1526 			q->u.in.polling = 0;  in handle_inbound()
1530 	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);  in handle_inbound()
1531 	atomic_add(count, &q->nr_buf_used);  in handle_inbound()
1533 	if (need_siga_in(q))  in handle_inbound()
1534 		return qdio_siga_input(q);  in handle_inbound()
1546 static int handle_outbound(struct qdio_q *q, unsigned int callflags,  in handle_outbound()  argument
1552 	qperf_inc(q, outbound_call);  in handle_outbound()
1554 	count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);  in handle_outbound()
1555 	used = atomic_add_return(count, &q->nr_buf_used);  in handle_outbound()
1558 		qperf_inc(q, outbound_queue_full);  in handle_outbound()
1561 		q->u.out.pci_out_enabled = 1;  in handle_outbound()
1562 		qperf_inc(q, pci_request_int);  in handle_outbound()
1564 		q->u.out.pci_out_enabled = 0;  in handle_outbound()
1566 	if (queue_type(q) == QDIO_IQDIO_QFMT) {  in handle_outbound()
1570 		WARN_ON_ONCE(count > 1 && !multicast_outbound(q));  in handle_outbound()
1572 		phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);  in handle_outbound()
1574 		rc = qdio_kick_outbound_q(q, phys_aob);  in handle_outbound()
1575 	} else if (need_siga_sync(q)) {  in handle_outbound()
1576 		rc = qdio_siga_sync_q(q);  in handle_outbound()
1579 		get_buf_state(q, prev_buf(bufnr), &state, 0);  in handle_outbound()
1581 			rc = qdio_kick_outbound_q(q, 0);  in handle_outbound()
1583 			qperf_inc(q, fast_requeue);  in handle_outbound()
1587 	if (used >= q->u.out.scan_threshold || rc)  in handle_outbound()
1588 		tasklet_schedule(&q->tasklet);  in handle_outbound()
1591 		if (!timer_pending(&q->u.out.timer))  in handle_outbound()
1592 			mod_timer(&q->u.out.timer, jiffies + HZ);  in handle_outbound()
1644 	struct qdio_q *q;  in qdio_start_irq()  local
1649 	q = irq_ptr->input_qs[nr];  in qdio_start_irq()
1652 	qdio_stop_polling(q);  in qdio_start_irq()
1653 	clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);  in qdio_start_irq()
1661 	if (!qdio_inbound_q_done(q))  in qdio_start_irq()
1667 			     &q->u.in.queue_irq_state))  in qdio_start_irq()
1690 	struct qdio_q *q;  in qdio_get_next_buffers()  local
1696 	q = irq_ptr->input_qs[nr];  in qdio_get_next_buffers()
1702 	if (need_siga_sync(q))  in qdio_get_next_buffers()
1703 		qdio_sync_queues(q);  in qdio_get_next_buffers()
1706 	qdio_check_outbound_after_thinint(q);  in qdio_get_next_buffers()
1708 	if (!qdio_inbound_q_moved(q))  in qdio_get_next_buffers()
1712 	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))  in qdio_get_next_buffers()
1715 	start = q->first_to_kick;  in qdio_get_next_buffers()
1716 	end = q->first_to_check;  in qdio_get_next_buffers()
1718 	*error = q->qdio_error;  in qdio_get_next_buffers()
1721 	q->first_to_kick = end;  in qdio_get_next_buffers()
1722 	q->qdio_error = 0;  in qdio_get_next_buffers()
1738 	struct qdio_q *q;  in qdio_stop_irq()  local
1743 	q = irq_ptr->input_qs[nr];  in qdio_stop_irq()
1746 			     &q->u.in.queue_irq_state))  in qdio_stop_irq()