118 static void fq_flow_set_detached(struct fq_flow *f)  in fq_flow_set_detached()  argument
120 	f->next = &detached;  in fq_flow_set_detached()
121 	f->age = jiffies;  in fq_flow_set_detached()
124 static bool fq_flow_is_detached(const struct fq_flow *f)  in fq_flow_is_detached()  argument
126 	return f->next == &detached;  in fq_flow_is_detached()
129 static void fq_flow_set_throttled(struct fq_sched_data *q, struct fq_flow *f)  in fq_flow_set_throttled()  argument
138 		if (f->time_next_packet >= aux->time_next_packet)  in fq_flow_set_throttled()
143 	rb_link_node(&f->rate_node, parent, p);  in fq_flow_set_throttled()
144 	rb_insert_color(&f->rate_node, &q->delayed);  in fq_flow_set_throttled()
148 	f->next = &throttled;  in fq_flow_set_throttled()
149 	if (q->time_next_delayed_flow > f->time_next_packet)  in fq_flow_set_throttled()
150 		q->time_next_delayed_flow = f->time_next_packet;  in fq_flow_set_throttled()
170 static bool fq_gc_candidate(const struct fq_flow *f)  in fq_gc_candidate()  argument
172 	return fq_flow_is_detached(f) &&  in fq_gc_candidate()
173 	       time_after(jiffies, f->age + FQ_GC_AGE);  in fq_gc_candidate()
180 	struct fq_flow *f, *tofree[FQ_GC_MAX];  in fq_gc()  local
189 		f = container_of(parent, struct fq_flow, fq_node);  in fq_gc()
190 		if (f->sk == sk)  in fq_gc()
193 		if (fq_gc_candidate(f)) {  in fq_gc()
194 			tofree[fcnt++] = f;  in fq_gc()
199 		if (f->sk > sk)  in fq_gc()
209 		struct fq_flow *f = tofree[--fcnt];  in fq_gc()  local
211 		rb_erase(&f->fq_node, root);  in fq_gc()
212 		kmem_cache_free(fq_flow_cachep, f);  in fq_gc()
221 	struct fq_flow *f;  in fq_classify()  local
257 		f = container_of(parent, struct fq_flow, fq_node);  in fq_classify()
258 		if (f->sk == sk) {  in fq_classify()
265 				     f->socket_hash != sk->sk_hash)) {  in fq_classify()
266 				f->credit = q->initial_quantum;  in fq_classify()
267 				f->socket_hash = sk->sk_hash;  in fq_classify()
268 				f->time_next_packet = 0ULL;  in fq_classify()
270 			return f;  in fq_classify()
272 		if (f->sk > sk)  in fq_classify()
278 	f = kmem_cache_zalloc(fq_flow_cachep, GFP_ATOMIC | __GFP_NOWARN);  in fq_classify()
279 	if (unlikely(!f)) {  in fq_classify()
283 	fq_flow_set_detached(f);  in fq_classify()
284 	f->sk = sk;  in fq_classify()
286 		f->socket_hash = sk->sk_hash;  in fq_classify()
287 	f->credit = q->initial_quantum;  in fq_classify()
289 	rb_link_node(&f->fq_node, parent, p);  in fq_classify()
290 	rb_insert_color(&f->fq_node, root);  in fq_classify()
294 	return f;  in fq_classify()
374 	struct fq_flow *f;  in fq_enqueue()  local
379 	f = fq_classify(skb, q);  in fq_enqueue()
380 	if (unlikely(f->qlen >= q->flow_plimit && f != &q->internal)) {  in fq_enqueue()
385 	f->qlen++;  in fq_enqueue()
389 	if (fq_flow_is_detached(f)) {  in fq_enqueue()
390 		fq_flow_add_tail(&q->new_flows, f);  in fq_enqueue()
391 		if (time_after(jiffies, f->age + q->flow_refill_delay))  in fq_enqueue()
392 			f->credit = max_t(u32, f->credit, q->quantum);  in fq_enqueue()
397 	flow_queue_add(f, skb);  in fq_enqueue()
399 	if (unlikely(f == &q->internal)) {  in fq_enqueue()
416 		struct fq_flow *f = container_of(p, struct fq_flow, rate_node);  in fq_check_throttled()  local
418 		if (f->time_next_packet > now) {  in fq_check_throttled()
419 			q->time_next_delayed_flow = f->time_next_packet;  in fq_check_throttled()
424 		fq_flow_add_tail(&q->old_flows, f);  in fq_check_throttled()
434 	struct fq_flow *f;  in fq_dequeue()  local
453 	f = head->first;  in fq_dequeue()
455 	if (f->credit <= 0) {  in fq_dequeue()
456 		f->credit += q->quantum;  in fq_dequeue()
457 		head->first = f->next;  in fq_dequeue()
458 		fq_flow_add_tail(&q->old_flows, f);  in fq_dequeue()
462 	skb = f->head;  in fq_dequeue()
463 	if (unlikely(skb && now < f->time_next_packet &&  in fq_dequeue()
465 		head->first = f->next;  in fq_dequeue()
466 		fq_flow_set_throttled(q, f);  in fq_dequeue()
470 	skb = fq_dequeue_head(sch, f);  in fq_dequeue()
472 		head->first = f->next;  in fq_dequeue()
475 			fq_flow_add_tail(&q->old_flows, f);  in fq_dequeue()
477 			fq_flow_set_detached(f);  in fq_dequeue()
483 	f->credit -= qdisc_pkt_len(skb);  in fq_dequeue()
485 	if (f->credit > 0 || !q->rate_enable)  in fq_dequeue()
511 		f->time_next_packet = now + len;  in fq_dequeue()
524 	struct fq_flow *f;  in fq_reset()  local
536 			f = container_of(p, struct fq_flow, fq_node);  in fq_reset()
539 			while ((skb = fq_dequeue_head(sch, f)) != NULL)  in fq_reset()
542 			kmem_cache_free(fq_flow_cachep, f);  in fq_reset()