160 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_segment()  local
175 		ret = qdisc_enqueue(segs, q->qdisc);  in tbf_segment()
184 	sch->q.qlen += nb;  in tbf_segment()
193 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_enqueue()  local
196 	if (qdisc_pkt_len(skb) > q->max_size) {  in tbf_enqueue()
197 		if (skb_is_gso(skb) && skb_gso_mac_seglen(skb) <= q->max_size)  in tbf_enqueue()
201 	ret = qdisc_enqueue(skb, q->qdisc);  in tbf_enqueue()
208 	sch->q.qlen++;  in tbf_enqueue()
214 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_drop()  local
217 	if (q->qdisc->ops->drop && (len = q->qdisc->ops->drop(q->qdisc)) != 0) {  in tbf_drop()
218 		sch->q.qlen--;  in tbf_drop()
224 static bool tbf_peak_present(const struct tbf_sched_data *q)  in tbf_peak_present()  argument
226 	return q->peak.rate_bytes_ps;  in tbf_peak_present()
231 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dequeue()  local
234 	skb = q->qdisc->ops->peek(q->qdisc);  in tbf_dequeue()
243 		toks = min_t(s64, now - q->t_c, q->buffer);  in tbf_dequeue()
245 		if (tbf_peak_present(q)) {  in tbf_dequeue()
246 			ptoks = toks + q->ptokens;  in tbf_dequeue()
247 			if (ptoks > q->mtu)  in tbf_dequeue()
248 				ptoks = q->mtu;  in tbf_dequeue()
249 			ptoks -= (s64) psched_l2t_ns(&q->peak, len);  in tbf_dequeue()
251 		toks += q->tokens;  in tbf_dequeue()
252 		if (toks > q->buffer)  in tbf_dequeue()
253 			toks = q->buffer;  in tbf_dequeue()
254 		toks -= (s64) psched_l2t_ns(&q->rate, len);  in tbf_dequeue()
257 			skb = qdisc_dequeue_peeked(q->qdisc);  in tbf_dequeue()
261 			q->t_c = now;  in tbf_dequeue()
262 			q->tokens = toks;  in tbf_dequeue()
263 			q->ptokens = ptoks;  in tbf_dequeue()
264 			sch->q.qlen--;  in tbf_dequeue()
270 		qdisc_watchdog_schedule_ns(&q->watchdog,  in tbf_dequeue()
292 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_reset()  local
294 	qdisc_reset(q->qdisc);  in tbf_reset()
295 	sch->q.qlen = 0;  in tbf_reset()
296 	q->t_c = ktime_get_ns();  in tbf_reset()
297 	q->tokens = q->buffer;  in tbf_reset()
298 	q->ptokens = q->mtu;  in tbf_reset()
299 	qdisc_watchdog_cancel(&q->watchdog);  in tbf_reset()
315 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_change()  local
388 	if (q->qdisc != &noop_qdisc) {  in tbf_change()
389 		err = fifo_set_limit(q->qdisc, qopt->limit);  in tbf_change()
402 		qdisc_tree_decrease_qlen(q->qdisc, q->qdisc->q.qlen);  in tbf_change()
403 		qdisc_destroy(q->qdisc);  in tbf_change()
404 		q->qdisc = child;  in tbf_change()
406 	q->limit = qopt->limit;  in tbf_change()
408 		q->mtu = mtu;  in tbf_change()
410 		q->mtu = PSCHED_TICKS2NS(qopt->mtu);  in tbf_change()
411 	q->max_size = max_size;  in tbf_change()
413 		q->buffer = buffer;  in tbf_change()
415 		q->buffer = PSCHED_TICKS2NS(qopt->buffer);  in tbf_change()
416 	q->tokens = q->buffer;  in tbf_change()
417 	q->ptokens = q->mtu;  in tbf_change()
419 	memcpy(&q->rate, &rate, sizeof(struct psched_ratecfg));  in tbf_change()
420 	memcpy(&q->peak, &peak, sizeof(struct psched_ratecfg));  in tbf_change()
430 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_init()  local
435 	q->t_c = ktime_get_ns();  in tbf_init()
436 	qdisc_watchdog_init(&q->watchdog, sch);  in tbf_init()
437 	q->qdisc = &noop_qdisc;  in tbf_init()
444 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_destroy()  local
446 	qdisc_watchdog_cancel(&q->watchdog);  in tbf_destroy()
447 	qdisc_destroy(q->qdisc);  in tbf_destroy()
452 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dump()  local
456 	sch->qstats.backlog = q->qdisc->qstats.backlog;  in tbf_dump()
461 	opt.limit = q->limit;  in tbf_dump()
462 	psched_ratecfg_getrate(&opt.rate, &q->rate);  in tbf_dump()
463 	if (tbf_peak_present(q))  in tbf_dump()
464 		psched_ratecfg_getrate(&opt.peakrate, &q->peak);  in tbf_dump()
467 	opt.mtu = PSCHED_NS2TICKS(q->mtu);  in tbf_dump()
468 	opt.buffer = PSCHED_NS2TICKS(q->buffer);  in tbf_dump()
471 	if (q->rate.rate_bytes_ps >= (1ULL << 32) &&  in tbf_dump()
472 	    nla_put_u64(skb, TCA_TBF_RATE64, q->rate.rate_bytes_ps))  in tbf_dump()
474 	if (tbf_peak_present(q) &&  in tbf_dump()
475 	    q->peak.rate_bytes_ps >= (1ULL << 32) &&  in tbf_dump()
476 	    nla_put_u64(skb, TCA_TBF_PRATE64, q->peak.rate_bytes_ps))  in tbf_dump()
489 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dump_class()  local
492 	tcm->tcm_info = q->qdisc->handle;  in tbf_dump_class()
500 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_graft()  local
506 	*old = q->qdisc;  in tbf_graft()
507 	q->qdisc = new;  in tbf_graft()
508 	qdisc_tree_decrease_qlen(*old, (*old)->q.qlen);  in tbf_graft()
517 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_leaf()  local
518 	return q->qdisc;  in tbf_leaf()