35 static struct cxl_sste* find_free_sste(struct cxl_context *ctx,  in find_free_sste()  argument
39 	unsigned int mask = (ctx->sst_size >> 7) - 1; /* SSTP0[SegTableSize] */  in find_free_sste()
48 	primary = ctx->sstp + (hash << 3);  in find_free_sste()
60 	ret = primary + ctx->sst_lru;  in find_free_sste()
61 	ctx->sst_lru = (ctx->sst_lru + 1) & 0x7;  in find_free_sste()
66 static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb)  in cxl_load_segment()  argument
72 	spin_lock_irqsave(&ctx->sste_lock, flags);  in cxl_load_segment()
73 	sste = find_free_sste(ctx, slb);  in cxl_load_segment()
78 			sste - ctx->sstp, slb->vsid, slb->esid);  in cxl_load_segment()
79 	trace_cxl_ste_write(ctx, sste - ctx->sstp, slb->esid, slb->vsid);  in cxl_load_segment()
84 	spin_unlock_irqrestore(&ctx->sste_lock, flags);  in cxl_load_segment()
87 static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm,  in cxl_fault_segment()  argument
94 		cxl_load_segment(ctx, &slb);  in cxl_fault_segment()
100 static void cxl_ack_ae(struct cxl_context *ctx)  in cxl_ack_ae()  argument
104 	cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0);  in cxl_ack_ae()
106 	spin_lock_irqsave(&ctx->lock, flags);  in cxl_ack_ae()
107 	ctx->pending_fault = true;  in cxl_ack_ae()
108 	ctx->fault_addr = ctx->dar;  in cxl_ack_ae()
109 	ctx->fault_dsisr = ctx->dsisr;  in cxl_ack_ae()
110 	spin_unlock_irqrestore(&ctx->lock, flags);  in cxl_ack_ae()
112 	wake_up_all(&ctx->wq);  in cxl_ack_ae()
115 static int cxl_handle_segment_miss(struct cxl_context *ctx,  in cxl_handle_segment_miss()  argument
120 	pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea);  in cxl_handle_segment_miss()
121 	trace_cxl_ste_miss(ctx, ea);  in cxl_handle_segment_miss()
123 	if ((rc = cxl_fault_segment(ctx, mm, ea)))  in cxl_handle_segment_miss()
124 		cxl_ack_ae(ctx);  in cxl_handle_segment_miss()
128 		cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);  in cxl_handle_segment_miss()
134 static void cxl_handle_page_fault(struct cxl_context *ctx,  in cxl_handle_page_fault()  argument
141 	trace_cxl_pte_miss(ctx, dsisr, dar);  in cxl_handle_page_fault()
145 		return cxl_ack_ae(ctx);  in cxl_handle_page_fault()
155 	if ((!ctx->kernel) || ~(dar & (1ULL << 63)))  in cxl_handle_page_fault()
165 	pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);  in cxl_handle_page_fault()
166 	cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);  in cxl_handle_page_fault()
171 	struct cxl_context *ctx =  in cxl_handle_fault()  local
173 	u64 dsisr = ctx->dsisr;  in cxl_handle_fault()
174 	u64 dar = ctx->dar;  in cxl_handle_fault()
178 	if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr ||  in cxl_handle_fault()
179 	    cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar ||  in cxl_handle_fault()
180 	    cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) {  in cxl_handle_fault()
184 		dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n");  in cxl_handle_fault()
189 	if (ctx->status == CLOSED) {  in cxl_handle_fault()
190 		cxl_ack_ae(ctx);  in cxl_handle_fault()
195 		"DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar);  in cxl_handle_fault()
197 	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {  in cxl_handle_fault()
199 			 pid_nr(ctx->pid));  in cxl_handle_fault()
200 		cxl_ack_ae(ctx);  in cxl_handle_fault()
205 			 pid_nr(ctx->pid));  in cxl_handle_fault()
206 		cxl_ack_ae(ctx);  in cxl_handle_fault()
211 		cxl_handle_segment_miss(ctx, mm, dar);  in cxl_handle_fault()
213 		cxl_handle_page_fault(ctx, mm, dsisr, dar);  in cxl_handle_fault()
222 static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)  in cxl_prefault_one()  argument
228 	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {  in cxl_prefault_one()
230 			 pid_nr(ctx->pid));  in cxl_prefault_one()
235 			 pid_nr(ctx->pid));  in cxl_prefault_one()
240 	rc = cxl_fault_segment(ctx, mm, ea);  in cxl_prefault_one()
256 static void cxl_prefault_vma(struct cxl_context *ctx)  in cxl_prefault_vma()  argument
265 	if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {  in cxl_prefault_vma()
267 			 pid_nr(ctx->pid));  in cxl_prefault_vma()
272 			 pid_nr(ctx->pid));  in cxl_prefault_vma()
287 			cxl_load_segment(ctx, &slb);  in cxl_prefault_vma()
298 void cxl_prefault(struct cxl_context *ctx, u64 wed)  in cxl_prefault()  argument
300 	switch (ctx->afu->prefault_mode) {  in cxl_prefault()
302 		cxl_prefault_one(ctx, wed);  in cxl_prefault()
305 		cxl_prefault_vma(ctx);  in cxl_prefault()