37 static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,  in initialize()  argument
44 	BUG_ON(!kq || !dev);  in initialize()
54 	kq->dev = dev;  in initialize()
55 	kq->nop_packet = nop.u32all;  in initialize()
59 		kq->mqd = dev->dqm->ops.get_mqd_manager(dev->dqm,  in initialize()
67 	if (kq->mqd == NULL)  in initialize()
77 	retval = kfd_gtt_sa_allocate(dev, queue_size, &kq->pq);  in initialize()
83 	kq->pq_kernel_addr = kq->pq->cpu_ptr;  in initialize()
84 	kq->pq_gpu_addr = kq->pq->gpu_addr;  in initialize()
86 	retval = kq->ops_asic_specific.initialize(kq, dev, type, queue_size);  in initialize()
90 	retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel),  in initialize()
91 					&kq->rptr_mem);  in initialize()
96 	kq->rptr_kernel = kq->rptr_mem->cpu_ptr;  in initialize()
97 	kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr;  in initialize()
99 	retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->wptr_kernel),  in initialize()
100 					&kq->wptr_mem);  in initialize()
105 	kq->wptr_kernel = kq->wptr_mem->cpu_ptr;  in initialize()
106 	kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr;  in initialize()
108 	memset(kq->pq_kernel_addr, 0, queue_size);  in initialize()
109 	memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel));  in initialize()
110 	memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel));  in initialize()
118 	prop.queue_address = kq->pq_gpu_addr;  in initialize()
119 	prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr;  in initialize()
120 	prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;  in initialize()
121 	prop.eop_ring_buffer_address = kq->eop_gpu_addr;  in initialize()
124 	if (init_queue(&kq->queue, prop) != 0)  in initialize()
127 	kq->queue->device = dev;  in initialize()
128 	kq->queue->process = kfd_get_process(current);  in initialize()
130 	retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd,  in initialize()
131 					&kq->queue->mqd_mem_obj,  in initialize()
132 					&kq->queue->gart_mqd_addr,  in initialize()
133 					&kq->queue->properties);  in initialize()
140 		kq->queue->pipe = KFD_CIK_HIQ_PIPE;  in initialize()
141 		kq->queue->queue = KFD_CIK_HIQ_QUEUE;  in initialize()
142 		kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,  in initialize()
143 					kq->queue->queue, NULL);  in initialize()
148 						&kq->fence_mem_obj);  in initialize()
153 		kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr;  in initialize()
154 		kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr;  in initialize()
157 	print_queue(kq->queue);  in initialize()
162 	uninit_queue(kq->queue);  in initialize()
164 	kfd_gtt_sa_free(dev, kq->wptr_mem);  in initialize()
166 	kfd_gtt_sa_free(dev, kq->rptr_mem);  in initialize()
168 	kfd_gtt_sa_free(dev, kq->eop_mem);  in initialize()
170 	kfd_gtt_sa_free(dev, kq->pq);  in initialize()
178 static void uninitialize(struct kernel_queue *kq)  in uninitialize()  argument
180 	BUG_ON(!kq);  in uninitialize()
182 	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)  in uninitialize()
183 		kq->mqd->destroy_mqd(kq->mqd,  in uninitialize()
187 					kq->queue->pipe,  in uninitialize()
188 					kq->queue->queue);  in uninitialize()
189 	else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)  in uninitialize()
190 		kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);  in uninitialize()
192 	kq->mqd->uninit_mqd(kq->mqd, kq->queue->mqd, kq->queue->mqd_mem_obj);  in uninitialize()
194 	kfd_gtt_sa_free(kq->dev, kq->rptr_mem);  in uninitialize()
195 	kfd_gtt_sa_free(kq->dev, kq->wptr_mem);  in uninitialize()
196 	kq->ops_asic_specific.uninitialize(kq);  in uninitialize()
197 	kfd_gtt_sa_free(kq->dev, kq->pq);  in uninitialize()
198 	kfd_release_kernel_doorbell(kq->dev,  in uninitialize()
199 					kq->queue->properties.doorbell_ptr);  in uninitialize()
200 	uninit_queue(kq->queue);  in uninitialize()
203 static int acquire_packet_buffer(struct kernel_queue *kq,  in acquire_packet_buffer()  argument
211 	BUG_ON(!kq || !buffer_ptr);  in acquire_packet_buffer()
213 	rptr = *kq->rptr_kernel;  in acquire_packet_buffer()
214 	wptr = *kq->wptr_kernel;  in acquire_packet_buffer()
215 	queue_address = (unsigned int *)kq->pq_kernel_addr;  in acquire_packet_buffer()
216 	queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t);  in acquire_packet_buffer()
237 			queue_address[wptr] = kq->nop_packet;  in acquire_packet_buffer()
243 	kq->pending_wptr = wptr + packet_size_in_dwords;  in acquire_packet_buffer()
248 static void submit_packet(struct kernel_queue *kq)  in submit_packet()  argument
254 	BUG_ON(!kq);  in submit_packet()
257 	for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) {  in submit_packet()
258 		pr_debug("0x%2X ", kq->pq_kernel_addr[i]);  in submit_packet()
265 	*kq->wptr_kernel = kq->pending_wptr;  in submit_packet()
266 	write_kernel_doorbell(kq->queue->properties.doorbell_ptr,  in submit_packet()
267 				kq->pending_wptr);  in submit_packet()
270 static void rollback_packet(struct kernel_queue *kq)  in rollback_packet()  argument
272 	BUG_ON(!kq);  in rollback_packet()
273 	kq->pending_wptr = *kq->queue->properties.write_ptr;  in rollback_packet()
279 	struct kernel_queue *kq;  in kernel_queue_init()  local
283 	kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL);  in kernel_queue_init()
284 	if (!kq)  in kernel_queue_init()
287 	kq->ops.initialize = initialize;  in kernel_queue_init()
288 	kq->ops.uninitialize = uninitialize;  in kernel_queue_init()
289 	kq->ops.acquire_packet_buffer = acquire_packet_buffer;  in kernel_queue_init()
290 	kq->ops.submit_packet = submit_packet;  in kernel_queue_init()
291 	kq->ops.rollback_packet = rollback_packet;  in kernel_queue_init()
295 		kernel_queue_init_vi(&kq->ops_asic_specific);  in kernel_queue_init()
299 		kernel_queue_init_cik(&kq->ops_asic_specific);  in kernel_queue_init()
303 	if (kq->ops.initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE) == false) {  in kernel_queue_init()
305 		kfree(kq);  in kernel_queue_init()
308 	return kq;  in kernel_queue_init()
311 void kernel_queue_uninit(struct kernel_queue *kq)  in kernel_queue_uninit()  argument
313 	BUG_ON(!kq);  in kernel_queue_uninit()
315 	kq->ops.uninitialize(kq);  in kernel_queue_uninit()
316 	kfree(kq);  in kernel_queue_uninit()
321 	struct kernel_queue *kq;  in test_kq()  local
329 	kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);  in test_kq()
330 	BUG_ON(!kq);  in test_kq()
332 	retval = kq->ops.acquire_packet_buffer(kq, 5, &buffer);  in test_kq()
335 		buffer[i] = kq->nop_packet;  in test_kq()
336 	kq->ops.submit_packet(kq);  in test_kq()