97 static int gru_wrap_asid(struct gru_state *gru)  in gru_wrap_asid()  argument
99 	gru_dbg(grudev, "gid %d\n", gru->gs_gid);  in gru_wrap_asid()
101 	gru->gs_asid_gen++;  in gru_wrap_asid()
106 static int gru_reset_asid_limit(struct gru_state *gru, int asid)  in gru_reset_asid_limit()  argument
110 	gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);  in gru_reset_asid_limit()
114 		asid = gru_wrap_asid(gru);  in gru_reset_asid_limit()
115 	gru_flush_all_tlb(gru);  in gru_reset_asid_limit()
116 	gid = gru->gs_gid;  in gru_reset_asid_limit()
119 		if (!gru->gs_gts[i] || is_kernel_context(gru->gs_gts[i]))  in gru_reset_asid_limit()
121 		inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid;  in gru_reset_asid_limit()
123 			gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms,  in gru_reset_asid_limit()
134 					asid = gru_wrap_asid(gru);  in gru_reset_asid_limit()
142 	gru->gs_asid_limit = limit;  in gru_reset_asid_limit()
143 	gru->gs_asid = asid;  in gru_reset_asid_limit()
144 	gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid,  in gru_reset_asid_limit()
150 static int gru_assign_asid(struct gru_state *gru)  in gru_assign_asid()  argument
154 	gru->gs_asid += ASID_INC;  in gru_assign_asid()
155 	asid = gru->gs_asid;  in gru_assign_asid()
156 	if (asid >= gru->gs_asid_limit)  in gru_assign_asid()
157 		asid = gru_reset_asid_limit(gru, asid);  in gru_assign_asid()
159 	gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);  in gru_assign_asid()
185 unsigned long gru_reserve_cb_resources(struct gru_state *gru, int cbr_au_count,  in gru_reserve_cb_resources()  argument
188 	return reserve_resources(&gru->gs_cbr_map, cbr_au_count, GRU_CBR_AU,  in gru_reserve_cb_resources()
192 unsigned long gru_reserve_ds_resources(struct gru_state *gru, int dsr_au_count,  in gru_reserve_ds_resources()  argument
195 	return reserve_resources(&gru->gs_dsr_map, dsr_au_count, GRU_DSR_AU,  in gru_reserve_ds_resources()
199 static void reserve_gru_resources(struct gru_state *gru,  in reserve_gru_resources()  argument
202 	gru->gs_active_contexts++;  in reserve_gru_resources()
204 	    gru_reserve_cb_resources(gru, gts->ts_cbr_au_count,  in reserve_gru_resources()
207 	    gru_reserve_ds_resources(gru, gts->ts_dsr_au_count, NULL);  in reserve_gru_resources()
210 static void free_gru_resources(struct gru_state *gru,  in free_gru_resources()  argument
213 	gru->gs_active_contexts--;  in free_gru_resources()
214 	gru->gs_cbr_map |= gts->ts_cbr_map;  in free_gru_resources()
215 	gru->gs_dsr_map |= gts->ts_dsr_map;  in free_gru_resources()
225 static int check_gru_resources(struct gru_state *gru, int cbr_au_count,  in check_gru_resources()  argument
228 	return hweight64(gru->gs_cbr_map) >= cbr_au_count  in check_gru_resources()
229 		&& hweight64(gru->gs_dsr_map) >= dsr_au_count  in check_gru_resources()
230 		&& gru->gs_active_contexts < max_active_contexts;  in check_gru_resources()
237 static int gru_load_mm_tracker(struct gru_state *gru,  in gru_load_mm_tracker()  argument
241 	struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid];  in gru_load_mm_tracker()
248 	spin_lock(&gru->gs_asid_lock);  in gru_load_mm_tracker()
250 			  gru->gs_asid_gen)) {  in gru_load_mm_tracker()
251 		asid = gru_assign_asid(gru);  in gru_load_mm_tracker()
253 		asids->mt_asid_gen = gru->gs_asid_gen;  in gru_load_mm_tracker()
258 	spin_unlock(&gru->gs_asid_lock);  in gru_load_mm_tracker()
262 	if (!test_bit(gru->gs_gid, gms->ms_asidmap))  in gru_load_mm_tracker()
263 		__set_bit(gru->gs_gid, gms->ms_asidmap);  in gru_load_mm_tracker()
268 		gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,  in gru_load_mm_tracker()
273 static void gru_unload_mm_tracker(struct gru_state *gru,  in gru_unload_mm_tracker()  argument
280 	asids = &gms->ms_asids[gru->gs_gid];  in gru_unload_mm_tracker()
283 	spin_lock(&gru->gs_asid_lock);  in gru_unload_mm_tracker()
287 		gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);  in gru_unload_mm_tracker()
288 	spin_unlock(&gru->gs_asid_lock);  in gru_unload_mm_tracker()
439 	struct gru_state *gru;  in gru_free_gru_context()  local
441 	gru = gts->ts_gru;  in gru_free_gru_context()
442 	gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);  in gru_free_gru_context()
444 	spin_lock(&gru->gs_lock);  in gru_free_gru_context()
445 	gru->gs_gts[gts->ts_ctxnum] = NULL;  in gru_free_gru_context()
446 	free_gru_resources(gru, gts);  in gru_free_gru_context()
447 	BUG_ON(test_bit(gts->ts_ctxnum, &gru->gs_context_map) == 0);  in gru_free_gru_context()
448 	__clear_bit(gts->ts_ctxnum, &gru->gs_context_map);  in gru_free_gru_context()
452 	spin_unlock(&gru->gs_lock);  in gru_free_gru_context()
557 	struct gru_state *gru = gts->ts_gru;  in gru_unload_context()  local
563 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);  in gru_unload_context()
572 		gru_unload_mm_tracker(gru, gts);  in gru_unload_context()
574 		gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr,  in gru_unload_context()
593 	struct gru_state *gru = gts->ts_gru;  in gru_load_context()  local
597 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);  in gru_load_context()
626 		asid = gru_load_mm_tracker(gru, gts);  in gru_load_context()
641 	gru_load_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, ctxnum,  in gru_load_context()
661 	struct gru_state *gru = gts->ts_gru;  in gru_update_cch()  local
664 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);  in gru_update_cch()
668 		if (gru->gs_gts[gts->ts_ctxnum] != gts)  in gru_update_cch()
712 static int gru_check_chiplet_assignment(struct gru_state *gru,  in gru_check_chiplet_assignment()  argument
723 	return gru->gs_blade_id == blade_id &&  in gru_check_chiplet_assignment()
724 		(chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id);  in gru_check_chiplet_assignment()
734 	struct gru_state *gru;  in gru_check_context_placement()  local
741 	gru = gts->ts_gru;  in gru_check_context_placement()
742 	if (!gru || gts->ts_tgid_owner != current->tgid)  in gru_check_context_placement()
745 	if (!gru_check_chiplet_assignment(gru, gts)) {  in gru_check_context_placement()
786 	struct gru_state *gru, *gru0;  in gru_steal_context()  local
801 	gru = blade->bs_lru_gru;  in gru_steal_context()
803 		gru = next_gru(blade, gru);  in gru_steal_context()
804 	blade->bs_lru_gru = gru;  in gru_steal_context()
807 	gru0 = gru;  in gru_steal_context()
809 		if (gru_check_chiplet_assignment(gru, gts)) {  in gru_steal_context()
810 			if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH))  in gru_steal_context()
812 			spin_lock(&gru->gs_lock);  in gru_steal_context()
814 				if (flag && gru == gru0 && ctxnum == ctxnum0)  in gru_steal_context()
816 				ngts = gru->gs_gts[ctxnum];  in gru_steal_context()
827 			spin_unlock(&gru->gs_lock);  in gru_steal_context()
828 			if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0))  in gru_steal_context()
831 		if (flag && gru == gru0)  in gru_steal_context()
835 		gru = next_gru(blade, gru);  in gru_steal_context()
850 		gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),  in gru_steal_context()
851 		hweight64(gru->gs_dsr_map));  in gru_steal_context()
857 static int gru_assign_context_number(struct gru_state *gru)  in gru_assign_context_number()  argument
861 	ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);  in gru_assign_context_number()
862 	__set_bit(ctxnum, &gru->gs_context_map);  in gru_assign_context_number()
871 	struct gru_state *gru, *grux;  in gru_assign_gru_context()  local
878 	gru = NULL;  in gru_assign_gru_context()
886 			gru = grux;  in gru_assign_gru_context()
893 	if (gru) {  in gru_assign_gru_context()
894 		spin_lock(&gru->gs_lock);  in gru_assign_gru_context()
895 		if (!check_gru_resources(gru, gts->ts_cbr_au_count,  in gru_assign_gru_context()
897 			spin_unlock(&gru->gs_lock);  in gru_assign_gru_context()
900 		reserve_gru_resources(gru, gts);  in gru_assign_gru_context()
901 		gts->ts_gru = gru;  in gru_assign_gru_context()
902 		gts->ts_blade = gru->gs_blade_id;  in gru_assign_gru_context()
903 		gts->ts_ctxnum = gru_assign_context_number(gru);  in gru_assign_gru_context()
905 		gru->gs_gts[gts->ts_ctxnum] = gts;  in gru_assign_gru_context()
906 		spin_unlock(&gru->gs_lock);  in gru_assign_gru_context()
919 	return gru;  in gru_assign_gru_context()