143 	struct qfq_aggregate *agg;	/* Parent aggregate. */  member
265 static void qfq_init_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_init_agg()  argument
268 	INIT_LIST_HEAD(&agg->active);  in qfq_init_agg()
269 	hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs);  in qfq_init_agg()
271 	agg->lmax = lmax;  in qfq_init_agg()
272 	agg->class_weight = weight;  in qfq_init_agg()
278 	struct qfq_aggregate *agg;  in qfq_find_agg()  local
280 	hlist_for_each_entry(agg, &q->nonfull_aggs, nonfull_next)  in qfq_find_agg()
281 		if (agg->lmax == lmax && agg->class_weight == weight)  in qfq_find_agg()
282 			return agg;  in qfq_find_agg()
289 static void qfq_update_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_update_agg()  argument
295 		hlist_del_init(&agg->nonfull_next);  in qfq_update_agg()
297 	if (agg->num_classes > new_num_classes &&  in qfq_update_agg()
299 		hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs);  in qfq_update_agg()
305 	agg->budgetmax = new_num_classes * agg->lmax;  in qfq_update_agg()
306 	new_agg_weight = agg->class_weight * new_num_classes;  in qfq_update_agg()
307 	agg->inv_w = ONE_FP/new_agg_weight;  in qfq_update_agg()
309 	if (agg->grp == NULL) {  in qfq_update_agg()
310 		int i = qfq_calc_index(agg->inv_w, agg->budgetmax,  in qfq_update_agg()
312 		agg->grp = &q->groups[i];  in qfq_update_agg()
316 		(int) agg->class_weight * (new_num_classes - agg->num_classes);  in qfq_update_agg()
319 	agg->num_classes = new_num_classes;  in qfq_update_agg()
324 			   struct qfq_aggregate *agg,  in qfq_add_to_agg()  argument
327 	cl->agg = agg;  in qfq_add_to_agg()
329 	qfq_update_agg(q, agg, agg->num_classes+1);  in qfq_add_to_agg()
331 		list_add_tail(&cl->alist, &agg->active);  in qfq_add_to_agg()
332 		if (list_first_entry(&agg->active, struct qfq_class, alist) ==  in qfq_add_to_agg()
333 		    cl && q->in_serv_agg != agg) /* agg was inactive */  in qfq_add_to_agg()
334 			qfq_activate_agg(q, agg, enqueue); /* schedule agg */  in qfq_add_to_agg()
340 static void qfq_destroy_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_destroy_agg()  argument
342 	if (!hlist_unhashed(&agg->nonfull_next))  in qfq_destroy_agg()
343 		hlist_del_init(&agg->nonfull_next);  in qfq_destroy_agg()
344 	q->wsum -= agg->class_weight;  in qfq_destroy_agg()
348 	if (q->in_serv_agg == agg)  in qfq_destroy_agg()
350 	kfree(agg);  in qfq_destroy_agg()
356 	struct qfq_aggregate *agg = cl->agg;  in qfq_deactivate_class()  local
360 	if (list_empty(&agg->active)) /* agg is now inactive */  in qfq_deactivate_class()
361 		qfq_deactivate_agg(q, agg);  in qfq_deactivate_class()
367 	struct qfq_aggregate *agg = cl->agg;  in qfq_rm_from_agg()  local
369 	cl->agg = NULL;  in qfq_rm_from_agg()
370 	if (agg->num_classes == 1) { /* agg being emptied, destroy it */  in qfq_rm_from_agg()
371 		qfq_destroy_agg(q, agg);  in qfq_rm_from_agg()
374 	qfq_update_agg(q, agg, agg->num_classes-1);  in qfq_rm_from_agg()
448 	    lmax == cl->agg->lmax &&  in qfq_change_class()
449 	    weight == cl->agg->class_weight)  in qfq_change_class()
452 	delta_w = weight - (cl ? cl->agg->class_weight : 0);  in qfq_change_class()
650 	if (nla_put_u32(skb, TCA_QFQ_WEIGHT, cl->agg->class_weight) ||  in qfq_dump_class()
651 	    nla_put_u32(skb, TCA_QFQ_LMAX, cl->agg->lmax))  in qfq_dump_class()
668 	xstats.weight = cl->agg->class_weight;  in qfq_dump_class_stats()
669 	xstats.lmax = cl->agg->lmax;  in qfq_dump_class_stats()
902 static void qfq_slot_insert(struct qfq_group *grp, struct qfq_aggregate *agg,  in qfq_slot_insert()  argument
911 		agg->S -= deltaS;  in qfq_slot_insert()
912 		agg->F -= deltaS;  in qfq_slot_insert()
918 	hlist_add_head(&agg->next, &grp->slots[i]);  in qfq_slot_insert()
934 	struct qfq_aggregate *agg = qfq_slot_head(grp);  in qfq_front_slot_remove()  local
936 	BUG_ON(!agg);  in qfq_front_slot_remove()
937 	hlist_del(&agg->next);  in qfq_front_slot_remove()
1000 static void agg_dequeue(struct qfq_aggregate *agg,  in agg_dequeue()  argument
1010 		cl->deficit += agg->lmax;  in agg_dequeue()
1011 		list_move_tail(&cl->alist, &agg->active);  in agg_dequeue()
1015 static inline struct sk_buff *qfq_peek_skb(struct qfq_aggregate *agg,  in qfq_peek_skb()  argument
1021 	*cl = list_first_entry(&agg->active, struct qfq_class, alist);  in qfq_peek_skb()
1032 static inline void charge_actual_service(struct qfq_aggregate *agg)  in charge_actual_service()  argument
1039 	u32 service_received = min(agg->budgetmax,  in charge_actual_service()
1040 				   agg->initial_budget - agg->budget);  in charge_actual_service()
1042 	agg->F = agg->S + (u64)service_received * agg->inv_w;  in charge_actual_service()
1057 static void qfq_update_start(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_update_start()  argument
1061 	int slot_shift = agg->grp->slot_shift;  in qfq_update_start()
1063 	roundedF = qfq_round_down(agg->F, slot_shift);  in qfq_update_start()
1066 	if (!qfq_gt(agg->F, q->V) || qfq_gt(roundedF, limit)) {  in qfq_update_start()
1068 		mask = mask_from(q->bitmaps[ER], agg->grp->index);  in qfq_update_start()
1073 					agg->S = next->F;  in qfq_update_start()
1075 					agg->S = limit;  in qfq_update_start()
1079 		agg->S = q->V;  in qfq_update_start()
1081 		agg->S = agg->F;  in qfq_update_start()
1091 		    struct qfq_aggregate *agg, enum update_reason reason)  in qfq_update_agg_ts()  argument
1094 		qfq_update_start(q, agg);  in qfq_update_agg_ts()
1096 		agg->S = agg->F;  in qfq_update_agg_ts()
1098 	agg->F = agg->S + (u64)agg->budgetmax * agg->inv_w;  in qfq_update_agg_ts()
1101 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg);
1182 	struct qfq_aggregate *agg, *new_front_agg;  in qfq_choose_next_agg()  local
1194 	agg = qfq_slot_head(grp);  in qfq_choose_next_agg()
1209 			return agg;  in qfq_choose_next_agg()
1219 	return agg;  in qfq_choose_next_agg()
1226 	struct qfq_aggregate *agg;  in qfq_enqueue()  local
1238 	if (unlikely(cl->agg->lmax < qdisc_pkt_len(skb))) {  in qfq_enqueue()
1240 			 cl->agg->lmax, qdisc_pkt_len(skb), cl->common.classid);  in qfq_enqueue()
1241 		err = qfq_change_agg(sch, cl, cl->agg->class_weight,  in qfq_enqueue()
1260 	agg = cl->agg;  in qfq_enqueue()
1264 		    list_first_entry(&agg->active, struct qfq_class, alist)  in qfq_enqueue()
1266 			list_move_tail(&cl->alist, &agg->active);  in qfq_enqueue()
1272 	cl->deficit = agg->lmax;  in qfq_enqueue()
1273 	list_add_tail(&cl->alist, &agg->active);  in qfq_enqueue()
1275 	if (list_first_entry(&agg->active, struct qfq_class, alist) != cl ||  in qfq_enqueue()
1276 	    q->in_serv_agg == agg)  in qfq_enqueue()
1279 	qfq_activate_agg(q, agg, enqueue);  in qfq_enqueue()
1287 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_schedule_agg()  argument
1289 	struct qfq_group *grp = agg->grp;  in qfq_schedule_agg()
1293 	roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_schedule_agg()
1305 		if (!qfq_gt(grp->S, agg->S))  in qfq_schedule_agg()
1324 		 (unsigned long long) agg->S,  in qfq_schedule_agg()
1325 		 (unsigned long long) agg->F,  in qfq_schedule_agg()
1329 	qfq_slot_insert(grp, agg, roundedS);  in qfq_schedule_agg()
1334 static void qfq_activate_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_activate_agg()  argument
1337 	agg->initial_budget = agg->budget = agg->budgetmax; /* recharge budg. */  in qfq_activate_agg()
1339 	qfq_update_agg_ts(q, agg, reason);  in qfq_activate_agg()
1341 		q->in_serv_agg = agg; /* start serving this aggregate */  in qfq_activate_agg()
1343 		q->oldV = q->V = agg->S;  in qfq_activate_agg()
1344 	} else if (agg != q->in_serv_agg)  in qfq_activate_agg()
1345 		qfq_schedule_agg(q, agg);  in qfq_activate_agg()
1349 			    struct qfq_aggregate *agg)  in qfq_slot_remove()  argument
1354 	roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_slot_remove()
1359 	hlist_del(&agg->next);  in qfq_slot_remove()
1371 static void qfq_deactivate_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_deactivate_agg()  argument
1373 	struct qfq_group *grp = agg->grp;  in qfq_deactivate_agg()
1378 	if (agg == q->in_serv_agg) {  in qfq_deactivate_agg()
1379 		charge_actual_service(agg);  in qfq_deactivate_agg()
1384 	agg->F = agg->S;  in qfq_deactivate_agg()
1385 	qfq_slot_remove(q, grp, agg);  in qfq_deactivate_agg()
1404 		agg = qfq_slot_scan(grp);  in qfq_deactivate_agg()
1405 		roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_deactivate_agg()
1431 	struct qfq_aggregate *agg;  in qfq_drop_from_slot()  local
1435 	hlist_for_each_entry(agg, slot, next) {  in qfq_drop_from_slot()
1436 		list_for_each_entry(cl, &agg->active, alist) {  in qfq_drop_from_slot()