160 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_segment()  local
177 		ret = qdisc_enqueue(segs, q->qdisc);  in tbf_segment()
186 	sch->q.qlen += nb;  in tbf_segment()
195 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_enqueue()  local
198 	if (qdisc_pkt_len(skb) > q->max_size) {  in tbf_enqueue()
199 		if (skb_is_gso(skb) && skb_gso_mac_seglen(skb) <= q->max_size)  in tbf_enqueue()
203 	ret = qdisc_enqueue(skb, q->qdisc);  in tbf_enqueue()
210 	sch->q.qlen++;  in tbf_enqueue()
216 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_drop()  local
219 	if (q->qdisc->ops->drop && (len = q->qdisc->ops->drop(q->qdisc)) != 0) {  in tbf_drop()
220 		sch->q.qlen--;  in tbf_drop()
226 static bool tbf_peak_present(const struct tbf_sched_data *q)  in tbf_peak_present()  argument
228 	return q->peak.rate_bytes_ps;  in tbf_peak_present()
233 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dequeue()  local
236 	skb = q->qdisc->ops->peek(q->qdisc);  in tbf_dequeue()
245 		toks = min_t(s64, now - q->t_c, q->buffer);  in tbf_dequeue()
247 		if (tbf_peak_present(q)) {  in tbf_dequeue()
248 			ptoks = toks + q->ptokens;  in tbf_dequeue()
249 			if (ptoks > q->mtu)  in tbf_dequeue()
250 				ptoks = q->mtu;  in tbf_dequeue()
251 			ptoks -= (s64) psched_l2t_ns(&q->peak, len);  in tbf_dequeue()
253 		toks += q->tokens;  in tbf_dequeue()
254 		if (toks > q->buffer)  in tbf_dequeue()
255 			toks = q->buffer;  in tbf_dequeue()
256 		toks -= (s64) psched_l2t_ns(&q->rate, len);  in tbf_dequeue()
259 			skb = qdisc_dequeue_peeked(q->qdisc);  in tbf_dequeue()
263 			q->t_c = now;  in tbf_dequeue()
264 			q->tokens = toks;  in tbf_dequeue()
265 			q->ptokens = ptoks;  in tbf_dequeue()
266 			sch->q.qlen--;  in tbf_dequeue()
272 		qdisc_watchdog_schedule_ns(&q->watchdog,  in tbf_dequeue()
294 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_reset()  local
296 	qdisc_reset(q->qdisc);  in tbf_reset()
297 	sch->q.qlen = 0;  in tbf_reset()
298 	q->t_c = ktime_get_ns();  in tbf_reset()
299 	q->tokens = q->buffer;  in tbf_reset()
300 	q->ptokens = q->mtu;  in tbf_reset()
301 	qdisc_watchdog_cancel(&q->watchdog);  in tbf_reset()
317 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_change()  local
390 	if (q->qdisc != &noop_qdisc) {  in tbf_change()
391 		err = fifo_set_limit(q->qdisc, qopt->limit);  in tbf_change()
404 		qdisc_tree_reduce_backlog(q->qdisc, q->qdisc->q.qlen,  in tbf_change()
405 					  q->qdisc->qstats.backlog);  in tbf_change()
406 		qdisc_destroy(q->qdisc);  in tbf_change()
407 		q->qdisc = child;  in tbf_change()
409 	q->limit = qopt->limit;  in tbf_change()
411 		q->mtu = mtu;  in tbf_change()
413 		q->mtu = PSCHED_TICKS2NS(qopt->mtu);  in tbf_change()
414 	q->max_size = max_size;  in tbf_change()
416 		q->buffer = buffer;  in tbf_change()
418 		q->buffer = PSCHED_TICKS2NS(qopt->buffer);  in tbf_change()
419 	q->tokens = q->buffer;  in tbf_change()
420 	q->ptokens = q->mtu;  in tbf_change()
422 	memcpy(&q->rate, &rate, sizeof(struct psched_ratecfg));  in tbf_change()
423 	memcpy(&q->peak, &peak, sizeof(struct psched_ratecfg));  in tbf_change()
433 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_init()  local
438 	q->t_c = ktime_get_ns();  in tbf_init()
439 	qdisc_watchdog_init(&q->watchdog, sch);  in tbf_init()
440 	q->qdisc = &noop_qdisc;  in tbf_init()
447 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_destroy()  local
449 	qdisc_watchdog_cancel(&q->watchdog);  in tbf_destroy()
450 	qdisc_destroy(q->qdisc);  in tbf_destroy()
455 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dump()  local
459 	sch->qstats.backlog = q->qdisc->qstats.backlog;  in tbf_dump()
464 	opt.limit = q->limit;  in tbf_dump()
465 	psched_ratecfg_getrate(&opt.rate, &q->rate);  in tbf_dump()
466 	if (tbf_peak_present(q))  in tbf_dump()
467 		psched_ratecfg_getrate(&opt.peakrate, &q->peak);  in tbf_dump()
470 	opt.mtu = PSCHED_NS2TICKS(q->mtu);  in tbf_dump()
471 	opt.buffer = PSCHED_NS2TICKS(q->buffer);  in tbf_dump()
474 	if (q->rate.rate_bytes_ps >= (1ULL << 32) &&  in tbf_dump()
475 	    nla_put_u64(skb, TCA_TBF_RATE64, q->rate.rate_bytes_ps))  in tbf_dump()
477 	if (tbf_peak_present(q) &&  in tbf_dump()
478 	    q->peak.rate_bytes_ps >= (1ULL << 32) &&  in tbf_dump()
479 	    nla_put_u64(skb, TCA_TBF_PRATE64, q->peak.rate_bytes_ps))  in tbf_dump()
492 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_dump_class()  local
495 	tcm->tcm_info = q->qdisc->handle;  in tbf_dump_class()
503 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_graft()  local
508 	*old = qdisc_replace(sch, new, &q->qdisc);  in tbf_graft()
514 	struct tbf_sched_data *q = qdisc_priv(sch);  in tbf_leaf()  local
515 	return q->qdisc;  in tbf_leaf()