143 	struct qfq_aggregate *agg;	/* Parent aggregate. */  member
265 static void qfq_init_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_init_agg()  argument
268 	INIT_LIST_HEAD(&agg->active);  in qfq_init_agg()
269 	hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs);  in qfq_init_agg()
271 	agg->lmax = lmax;  in qfq_init_agg()
272 	agg->class_weight = weight;  in qfq_init_agg()
278 	struct qfq_aggregate *agg;  in qfq_find_agg()  local
280 	hlist_for_each_entry(agg, &q->nonfull_aggs, nonfull_next)  in qfq_find_agg()
281 		if (agg->lmax == lmax && agg->class_weight == weight)  in qfq_find_agg()
282 			return agg;  in qfq_find_agg()
289 static void qfq_update_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_update_agg()  argument
295 		hlist_del_init(&agg->nonfull_next);  in qfq_update_agg()
297 	if (agg->num_classes > new_num_classes &&  in qfq_update_agg()
299 		hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs);  in qfq_update_agg()
305 	agg->budgetmax = new_num_classes * agg->lmax;  in qfq_update_agg()
306 	new_agg_weight = agg->class_weight * new_num_classes;  in qfq_update_agg()
307 	agg->inv_w = ONE_FP/new_agg_weight;  in qfq_update_agg()
309 	if (agg->grp == NULL) {  in qfq_update_agg()
310 		int i = qfq_calc_index(agg->inv_w, agg->budgetmax,  in qfq_update_agg()
312 		agg->grp = &q->groups[i];  in qfq_update_agg()
316 		(int) agg->class_weight * (new_num_classes - agg->num_classes);  in qfq_update_agg()
319 	agg->num_classes = new_num_classes;  in qfq_update_agg()
324 			   struct qfq_aggregate *agg,  in qfq_add_to_agg()  argument
327 	cl->agg = agg;  in qfq_add_to_agg()
329 	qfq_update_agg(q, agg, agg->num_classes+1);  in qfq_add_to_agg()
331 		list_add_tail(&cl->alist, &agg->active);  in qfq_add_to_agg()
332 		if (list_first_entry(&agg->active, struct qfq_class, alist) ==  in qfq_add_to_agg()
333 		    cl && q->in_serv_agg != agg) /* agg was inactive */  in qfq_add_to_agg()
334 			qfq_activate_agg(q, agg, enqueue); /* schedule agg */  in qfq_add_to_agg()
340 static void qfq_destroy_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_destroy_agg()  argument
342 	hlist_del_init(&agg->nonfull_next);  in qfq_destroy_agg()
343 	q->wsum -= agg->class_weight;  in qfq_destroy_agg()
347 	if (q->in_serv_agg == agg)  in qfq_destroy_agg()
349 	kfree(agg);  in qfq_destroy_agg()
355 	struct qfq_aggregate *agg = cl->agg;  in qfq_deactivate_class()  local
359 	if (list_empty(&agg->active)) /* agg is now inactive */  in qfq_deactivate_class()
360 		qfq_deactivate_agg(q, agg);  in qfq_deactivate_class()
366 	struct qfq_aggregate *agg = cl->agg;  in qfq_rm_from_agg()  local
368 	cl->agg = NULL;  in qfq_rm_from_agg()
369 	if (agg->num_classes == 1) { /* agg being emptied, destroy it */  in qfq_rm_from_agg()
370 		qfq_destroy_agg(q, agg);  in qfq_rm_from_agg()
373 	qfq_update_agg(q, agg, agg->num_classes-1);  in qfq_rm_from_agg()
447 	    lmax == cl->agg->lmax &&  in qfq_change_class()
448 	    weight == cl->agg->class_weight)  in qfq_change_class()
451 	delta_w = weight - (cl ? cl->agg->class_weight : 0);  in qfq_change_class()
645 	if (nla_put_u32(skb, TCA_QFQ_WEIGHT, cl->agg->class_weight) ||  in qfq_dump_class()
646 	    nla_put_u32(skb, TCA_QFQ_LMAX, cl->agg->lmax))  in qfq_dump_class()
663 	xstats.weight = cl->agg->class_weight;  in qfq_dump_class_stats()
664 	xstats.lmax = cl->agg->lmax;  in qfq_dump_class_stats()
897 static void qfq_slot_insert(struct qfq_group *grp, struct qfq_aggregate *agg,  in qfq_slot_insert()  argument
906 		agg->S -= deltaS;  in qfq_slot_insert()
907 		agg->F -= deltaS;  in qfq_slot_insert()
913 	hlist_add_head(&agg->next, &grp->slots[i]);  in qfq_slot_insert()
929 	struct qfq_aggregate *agg = qfq_slot_head(grp);  in qfq_front_slot_remove()  local
931 	BUG_ON(!agg);  in qfq_front_slot_remove()
932 	hlist_del(&agg->next);  in qfq_front_slot_remove()
995 static void agg_dequeue(struct qfq_aggregate *agg,  in agg_dequeue()  argument
1005 		cl->deficit += agg->lmax;  in agg_dequeue()
1006 		list_move_tail(&cl->alist, &agg->active);  in agg_dequeue()
1010 static inline struct sk_buff *qfq_peek_skb(struct qfq_aggregate *agg,  in qfq_peek_skb()  argument
1016 	*cl = list_first_entry(&agg->active, struct qfq_class, alist);  in qfq_peek_skb()
1027 static inline void charge_actual_service(struct qfq_aggregate *agg)  in charge_actual_service()  argument
1034 	u32 service_received = min(agg->budgetmax,  in charge_actual_service()
1035 				   agg->initial_budget - agg->budget);  in charge_actual_service()
1037 	agg->F = agg->S + (u64)service_received * agg->inv_w;  in charge_actual_service()
1052 static void qfq_update_start(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_update_start()  argument
1056 	int slot_shift = agg->grp->slot_shift;  in qfq_update_start()
1058 	roundedF = qfq_round_down(agg->F, slot_shift);  in qfq_update_start()
1061 	if (!qfq_gt(agg->F, q->V) || qfq_gt(roundedF, limit)) {  in qfq_update_start()
1063 		mask = mask_from(q->bitmaps[ER], agg->grp->index);  in qfq_update_start()
1068 					agg->S = next->F;  in qfq_update_start()
1070 					agg->S = limit;  in qfq_update_start()
1074 		agg->S = q->V;  in qfq_update_start()
1076 		agg->S = agg->F;  in qfq_update_start()
1086 		    struct qfq_aggregate *agg, enum update_reason reason)  in qfq_update_agg_ts()  argument
1089 		qfq_update_start(q, agg);  in qfq_update_agg_ts()
1091 		agg->S = agg->F;  in qfq_update_agg_ts()
1093 	agg->F = agg->S + (u64)agg->budgetmax * agg->inv_w;  in qfq_update_agg_ts()
1096 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg);
1177 	struct qfq_aggregate *agg, *new_front_agg;  in qfq_choose_next_agg()  local
1189 	agg = qfq_slot_head(grp);  in qfq_choose_next_agg()
1204 			return agg;  in qfq_choose_next_agg()
1214 	return agg;  in qfq_choose_next_agg()
1221 	struct qfq_aggregate *agg;  in qfq_enqueue()  local
1233 	if (unlikely(cl->agg->lmax < qdisc_pkt_len(skb))) {  in qfq_enqueue()
1235 			 cl->agg->lmax, qdisc_pkt_len(skb), cl->common.classid);  in qfq_enqueue()
1236 		err = qfq_change_agg(sch, cl, cl->agg->class_weight,  in qfq_enqueue()
1255 	agg = cl->agg;  in qfq_enqueue()
1259 		    list_first_entry(&agg->active, struct qfq_class, alist)  in qfq_enqueue()
1261 			list_move_tail(&cl->alist, &agg->active);  in qfq_enqueue()
1267 	cl->deficit = agg->lmax;  in qfq_enqueue()
1268 	list_add_tail(&cl->alist, &agg->active);  in qfq_enqueue()
1270 	if (list_first_entry(&agg->active, struct qfq_class, alist) != cl ||  in qfq_enqueue()
1271 	    q->in_serv_agg == agg)  in qfq_enqueue()
1274 	qfq_activate_agg(q, agg, enqueue);  in qfq_enqueue()
1282 static void qfq_schedule_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_schedule_agg()  argument
1284 	struct qfq_group *grp = agg->grp;  in qfq_schedule_agg()
1288 	roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_schedule_agg()
1300 		if (!qfq_gt(grp->S, agg->S))  in qfq_schedule_agg()
1319 		 (unsigned long long) agg->S,  in qfq_schedule_agg()
1320 		 (unsigned long long) agg->F,  in qfq_schedule_agg()
1324 	qfq_slot_insert(grp, agg, roundedS);  in qfq_schedule_agg()
1329 static void qfq_activate_agg(struct qfq_sched *q, struct qfq_aggregate *agg,  in qfq_activate_agg()  argument
1332 	agg->initial_budget = agg->budget = agg->budgetmax; /* recharge budg. */  in qfq_activate_agg()
1334 	qfq_update_agg_ts(q, agg, reason);  in qfq_activate_agg()
1336 		q->in_serv_agg = agg; /* start serving this aggregate */  in qfq_activate_agg()
1338 		q->oldV = q->V = agg->S;  in qfq_activate_agg()
1339 	} else if (agg != q->in_serv_agg)  in qfq_activate_agg()
1340 		qfq_schedule_agg(q, agg);  in qfq_activate_agg()
1344 			    struct qfq_aggregate *agg)  in qfq_slot_remove()  argument
1349 	roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_slot_remove()
1354 	hlist_del(&agg->next);  in qfq_slot_remove()
1366 static void qfq_deactivate_agg(struct qfq_sched *q, struct qfq_aggregate *agg)  in qfq_deactivate_agg()  argument
1368 	struct qfq_group *grp = agg->grp;  in qfq_deactivate_agg()
1373 	if (agg == q->in_serv_agg) {  in qfq_deactivate_agg()
1374 		charge_actual_service(agg);  in qfq_deactivate_agg()
1379 	agg->F = agg->S;  in qfq_deactivate_agg()
1380 	qfq_slot_remove(q, grp, agg);  in qfq_deactivate_agg()
1399 		agg = qfq_slot_scan(grp);  in qfq_deactivate_agg()
1400 		roundedS = qfq_round_down(agg->S, grp->slot_shift);  in qfq_deactivate_agg()
1426 	struct qfq_aggregate *agg;  in qfq_drop_from_slot()  local
1430 	hlist_for_each_entry(agg, slot, next) {  in qfq_drop_from_slot()
1431 		list_for_each_entry(cl, &agg->active, alist) {  in qfq_drop_from_slot()