118 static void fq_flow_set_detached(struct fq_flow *f)  in fq_flow_set_detached()  argument
120 	f->next = &detached;  in fq_flow_set_detached()
121 	f->age = jiffies;  in fq_flow_set_detached()
124 static bool fq_flow_is_detached(const struct fq_flow *f)  in fq_flow_is_detached()  argument
126 	return f->next == &detached;  in fq_flow_is_detached()
129 static void fq_flow_set_throttled(struct fq_sched_data *q, struct fq_flow *f)  in fq_flow_set_throttled()  argument
138 		if (f->time_next_packet >= aux->time_next_packet)  in fq_flow_set_throttled()
143 	rb_link_node(&f->rate_node, parent, p);  in fq_flow_set_throttled()
144 	rb_insert_color(&f->rate_node, &q->delayed);  in fq_flow_set_throttled()
148 	f->next = &throttled;  in fq_flow_set_throttled()
149 	if (q->time_next_delayed_flow > f->time_next_packet)  in fq_flow_set_throttled()
150 		q->time_next_delayed_flow = f->time_next_packet;  in fq_flow_set_throttled()
170 static bool fq_gc_candidate(const struct fq_flow *f)  in fq_gc_candidate()  argument
172 	return fq_flow_is_detached(f) &&  in fq_gc_candidate()
173 	       time_after(jiffies, f->age + FQ_GC_AGE);  in fq_gc_candidate()
180 	struct fq_flow *f, *tofree[FQ_GC_MAX];  in fq_gc()  local
189 		f = container_of(parent, struct fq_flow, fq_node);  in fq_gc()
190 		if (f->sk == sk)  in fq_gc()
193 		if (fq_gc_candidate(f)) {  in fq_gc()
194 			tofree[fcnt++] = f;  in fq_gc()
199 		if (f->sk > sk)  in fq_gc()
209 		struct fq_flow *f = tofree[--fcnt];  in fq_gc()  local
211 		rb_erase(&f->fq_node, root);  in fq_gc()
212 		kmem_cache_free(fq_flow_cachep, f);  in fq_gc()
221 	struct fq_flow *f;  in fq_classify()  local
254 		f = container_of(parent, struct fq_flow, fq_node);  in fq_classify()
255 		if (f->sk == sk) {  in fq_classify()
262 				     f->socket_hash != sk->sk_hash)) {  in fq_classify()
263 				f->credit = q->initial_quantum;  in fq_classify()
264 				f->socket_hash = sk->sk_hash;  in fq_classify()
265 				f->time_next_packet = 0ULL;  in fq_classify()
267 			return f;  in fq_classify()
269 		if (f->sk > sk)  in fq_classify()
275 	f = kmem_cache_zalloc(fq_flow_cachep, GFP_ATOMIC | __GFP_NOWARN);  in fq_classify()
276 	if (unlikely(!f)) {  in fq_classify()
280 	fq_flow_set_detached(f);  in fq_classify()
281 	f->sk = sk;  in fq_classify()
283 		f->socket_hash = sk->sk_hash;  in fq_classify()
284 	f->credit = q->initial_quantum;  in fq_classify()
286 	rb_link_node(&f->fq_node, parent, p);  in fq_classify()
287 	rb_insert_color(&f->fq_node, root);  in fq_classify()
291 	return f;  in fq_classify()
371 	struct fq_flow *f;  in fq_enqueue()  local
376 	f = fq_classify(skb, q);  in fq_enqueue()
377 	if (unlikely(f->qlen >= q->flow_plimit && f != &q->internal)) {  in fq_enqueue()
382 	f->qlen++;  in fq_enqueue()
386 	if (fq_flow_is_detached(f)) {  in fq_enqueue()
387 		fq_flow_add_tail(&q->new_flows, f);  in fq_enqueue()
388 		if (time_after(jiffies, f->age + q->flow_refill_delay))  in fq_enqueue()
389 			f->credit = max_t(u32, f->credit, q->quantum);  in fq_enqueue()
394 	flow_queue_add(f, skb);  in fq_enqueue()
396 	if (unlikely(f == &q->internal)) {  in fq_enqueue()
413 		struct fq_flow *f = container_of(p, struct fq_flow, rate_node);  in fq_check_throttled()  local
415 		if (f->time_next_packet > now) {  in fq_check_throttled()
416 			q->time_next_delayed_flow = f->time_next_packet;  in fq_check_throttled()
421 		fq_flow_add_tail(&q->old_flows, f);  in fq_check_throttled()
431 	struct fq_flow *f;  in fq_dequeue()  local
450 	f = head->first;  in fq_dequeue()
452 	if (f->credit <= 0) {  in fq_dequeue()
453 		f->credit += q->quantum;  in fq_dequeue()
454 		head->first = f->next;  in fq_dequeue()
455 		fq_flow_add_tail(&q->old_flows, f);  in fq_dequeue()
459 	skb = f->head;  in fq_dequeue()
460 	if (unlikely(skb && now < f->time_next_packet &&  in fq_dequeue()
462 		head->first = f->next;  in fq_dequeue()
463 		fq_flow_set_throttled(q, f);  in fq_dequeue()
467 	skb = fq_dequeue_head(sch, f);  in fq_dequeue()
469 		head->first = f->next;  in fq_dequeue()
472 			fq_flow_add_tail(&q->old_flows, f);  in fq_dequeue()
474 			fq_flow_set_detached(f);  in fq_dequeue()
480 	f->credit -= qdisc_pkt_len(skb);  in fq_dequeue()
482 	if (f->credit > 0 || !q->rate_enable)  in fq_dequeue()
508 		f->time_next_packet = now + len;  in fq_dequeue()
521 	struct fq_flow *f;  in fq_reset()  local
533 			f = container_of(p, struct fq_flow, fq_node);  in fq_reset()
536 			while ((skb = fq_dequeue_head(sch, f)) != NULL)  in fq_reset()
539 			kmem_cache_free(fq_flow_cachep, f);  in fq_reset()