70 static unsigned int fq_codel_hash(const struct fq_codel_sched_data *q,  in fq_codel_hash()  argument
79 			    (__force u32)keys.ports, q->perturbation);  in fq_codel_hash()
81 	return reciprocal_scale(hash, q->flows_cnt);  in fq_codel_hash()
87 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_classify()  local
94 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_codel_classify()
97 	filter = rcu_dereference_bh(q->filter_list);  in fq_codel_classify()
99 		return fq_codel_hash(q, skb) + 1;  in fq_codel_classify()
113 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_codel_classify()
145 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_drop()  local
155 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_drop()
156 		if (q->backlogs[i] > maxbacklog) {  in fq_codel_drop()
157 			maxbacklog = q->backlogs[i];  in fq_codel_drop()
161 	flow = &q->flows[idx];  in fq_codel_drop()
164 	q->backlogs[idx] -= len;  in fq_codel_drop()
165 	sch->q.qlen--;  in fq_codel_drop()
175 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_enqueue()  local
190 	flow = &q->flows[idx];  in fq_codel_enqueue()
192 	q->backlogs[idx] += qdisc_pkt_len(skb);  in fq_codel_enqueue()
196 		list_add_tail(&flow->flowchain, &q->new_flows);  in fq_codel_enqueue()
197 		q->new_flow_count++;  in fq_codel_enqueue()
198 		flow->deficit = q->quantum;  in fq_codel_enqueue()
201 	if (++sch->q.qlen <= sch->limit)  in fq_codel_enqueue()
204 	q->drop_overlimit++;  in fq_codel_enqueue()
222 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in dequeue()  local
229 		q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb);  in dequeue()
230 		sch->q.qlen--;  in dequeue()
237 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dequeue()  local
244 	head = &q->new_flows;  in fq_codel_dequeue()
246 		head = &q->old_flows;  in fq_codel_dequeue()
253 		flow->deficit += q->quantum;  in fq_codel_dequeue()
254 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_codel_dequeue()
258 	prev_drop_count = q->cstats.drop_count;  in fq_codel_dequeue()
259 	prev_ecn_mark = q->cstats.ecn_mark;  in fq_codel_dequeue()
261 	skb = codel_dequeue(sch, &q->cparams, &flow->cvars, &q->cstats,  in fq_codel_dequeue()
264 	flow->dropped += q->cstats.drop_count - prev_drop_count;  in fq_codel_dequeue()
265 	flow->dropped += q->cstats.ecn_mark - prev_ecn_mark;  in fq_codel_dequeue()
269 		if ((head == &q->new_flows) && !list_empty(&q->old_flows))  in fq_codel_dequeue()
270 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_codel_dequeue()
280 	if (q->cstats.drop_count && sch->q.qlen) {  in fq_codel_dequeue()
281 		qdisc_tree_decrease_qlen(sch, q->cstats.drop_count);  in fq_codel_dequeue()
282 		q->cstats.drop_count = 0;  in fq_codel_dequeue()
306 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_change()  local
317 		if (q->flows)  in fq_codel_change()
319 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_CODEL_FLOWS]);  in fq_codel_change()
320 		if (!q->flows_cnt ||  in fq_codel_change()
321 		    q->flows_cnt > 65536)  in fq_codel_change()
329 		q->cparams.target = (target * NSEC_PER_USEC) >> CODEL_SHIFT;  in fq_codel_change()
335 		q->cparams.interval = (interval * NSEC_PER_USEC) >> CODEL_SHIFT;  in fq_codel_change()
342 		q->cparams.ecn = !!nla_get_u32(tb[TCA_FQ_CODEL_ECN]);  in fq_codel_change()
345 		q->quantum = max(256U, nla_get_u32(tb[TCA_FQ_CODEL_QUANTUM]));  in fq_codel_change()
347 	while (sch->q.qlen > sch->limit) {  in fq_codel_change()
351 		q->cstats.drop_count++;  in fq_codel_change()
353 	qdisc_tree_decrease_qlen(sch, q->cstats.drop_count);  in fq_codel_change()
354 	q->cstats.drop_count = 0;  in fq_codel_change()
376 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_destroy()  local
378 	tcf_destroy_chain(&q->filter_list);  in fq_codel_destroy()
379 	fq_codel_free(q->backlogs);  in fq_codel_destroy()
380 	fq_codel_free(q->flows);  in fq_codel_destroy()
385 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_init()  local
389 	q->flows_cnt = 1024;  in fq_codel_init()
390 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_codel_init()
391 	q->perturbation = prandom_u32();  in fq_codel_init()
392 	INIT_LIST_HEAD(&q->new_flows);  in fq_codel_init()
393 	INIT_LIST_HEAD(&q->old_flows);  in fq_codel_init()
394 	codel_params_init(&q->cparams, sch);  in fq_codel_init()
395 	codel_stats_init(&q->cstats);  in fq_codel_init()
396 	q->cparams.ecn = true;  in fq_codel_init()
404 	if (!q->flows) {  in fq_codel_init()
405 		q->flows = fq_codel_zalloc(q->flows_cnt *  in fq_codel_init()
407 		if (!q->flows)  in fq_codel_init()
409 		q->backlogs = fq_codel_zalloc(q->flows_cnt * sizeof(u32));  in fq_codel_init()
410 		if (!q->backlogs) {  in fq_codel_init()
411 			fq_codel_free(q->flows);  in fq_codel_init()
414 		for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_init()
415 			struct fq_codel_flow *flow = q->flows + i;  in fq_codel_init()
430 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump()  local
438 			codel_time_to_us(q->cparams.target)) ||  in fq_codel_dump()
442 			codel_time_to_us(q->cparams.interval)) ||  in fq_codel_dump()
444 			q->cparams.ecn) ||  in fq_codel_dump()
446 			q->quantum) ||  in fq_codel_dump()
448 			q->flows_cnt))  in fq_codel_dump()
459 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump_stats()  local
465 	st.qdisc_stats.maxpacket = q->cstats.maxpacket;  in fq_codel_dump_stats()
466 	st.qdisc_stats.drop_overlimit = q->drop_overlimit;  in fq_codel_dump_stats()
467 	st.qdisc_stats.ecn_mark = q->cstats.ecn_mark;  in fq_codel_dump_stats()
468 	st.qdisc_stats.new_flow_count = q->new_flow_count;  in fq_codel_dump_stats()
470 	list_for_each(pos, &q->new_flows)  in fq_codel_dump_stats()
473 	list_for_each(pos, &q->old_flows)  in fq_codel_dump_stats()
497 static void fq_codel_put(struct Qdisc *q, unsigned long cl)  in fq_codel_put()  argument
504 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_find_tcf()  local
508 	return &q->filter_list;  in fq_codel_find_tcf()
521 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_dump_class_stats()  local
526 	if (idx < q->flows_cnt) {  in fq_codel_dump_class_stats()
527 		const struct fq_codel_flow *flow = &q->flows[idx];  in fq_codel_dump_class_stats()
550 		qs.backlog = q->backlogs[idx];  in fq_codel_dump_class_stats()
555 	if (idx < q->flows_cnt)  in fq_codel_dump_class_stats()
562 	struct fq_codel_sched_data *q = qdisc_priv(sch);  in fq_codel_walk()  local
568 	for (i = 0; i < q->flows_cnt; i++) {  in fq_codel_walk()
569 		if (list_empty(&q->flows[i].flowchain) ||  in fq_codel_walk()