dqm 108 drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c kfd_process_vm_fault(dev->dqm, pasid); dqm 504 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, dqm 538 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c if (dev->dqm->ops.set_trap_handler(dev->dqm, dqm 1130 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS && dqm 87 drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c if (pdev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) dqm 677 drivers/gpu/drm/amd/amdkfd/kfd_device.c kfd->dqm = device_queue_manager_init(kfd); dqm 678 drivers/gpu/drm/amd/amdkfd/kfd_device.c if (!kfd->dqm) { dqm 705 drivers/gpu/drm/amd/amdkfd/kfd_device.c kfd->dqm->sched_policy); dqm 712 drivers/gpu/drm/amd/amdkfd/kfd_device.c device_queue_manager_uninit(kfd->dqm); dqm 735 drivers/gpu/drm/amd/amdkfd/kfd_device.c device_queue_manager_uninit(kfd->dqm); dqm 755 drivers/gpu/drm/amd/amdkfd/kfd_device.c dqm_lock(kfd->dqm); dqm 774 drivers/gpu/drm/amd/amdkfd/kfd_device.c dqm_unlock(kfd->dqm); dqm 800 drivers/gpu/drm/amd/amdkfd/kfd_device.c kfd->dqm->ops.stop(kfd->dqm); dqm 836 drivers/gpu/drm/amd/amdkfd/kfd_device.c err = kfd->dqm->ops.start(kfd->dqm); dqm 1176 drivers/gpu/drm/amd/amdkfd/kfd_device.c if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) { dqm 1181 drivers/gpu/drm/amd/amdkfd/kfd_device.c r = pm_debugfs_hang_hws(&dev->dqm->packets); dqm 1183 drivers/gpu/drm/amd/amdkfd/kfd_device.c r = dqm_debugfs_execute_queues(dev->dqm); dqm 42 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, dqm 45 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int execute_queues_cpsch(struct device_queue_manager *dqm, dqm 48 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unmap_queues_cpsch(struct device_queue_manager *dqm, dqm 52 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm); dqm 54 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static void deallocate_sdma_queue(struct device_queue_manager *dqm, dqm 57 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static inline void deallocate_hqd(struct device_queue_manager *dqm, dqm 59 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q); dqm 60 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int allocate_sdma_queue(struct device_queue_manager *dqm, dqm 72 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe) dqm 75 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec dqm 76 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c + pipe * dqm->dev->shared_resources.num_queue_per_pipe; dqm 79 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i) dqm 81 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->shared_resources.queue_bitmap)) dqm 86 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unsigned int get_queues_num(struct device_queue_manager *dqm) dqm 88 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return bitmap_weight(dqm->dev->shared_resources.queue_bitmap, dqm 92 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unsigned int get_queues_per_pipe(struct device_queue_manager *dqm) dqm 94 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->shared_resources.num_queue_per_pipe; dqm 97 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) dqm 99 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->shared_resources.num_pipe_per_mec; dqm 102 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm) dqm 104 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->device_info->num_sdma_engines; dqm 107 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm) dqm 109 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->device_info->num_xgmi_sdma_engines; dqm 112 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) dqm 114 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->device_info->num_sdma_engines dqm 115 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c * dqm->dev->device_info->num_sdma_queues_per_engine; dqm 118 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) dqm 120 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->device_info->num_xgmi_sdma_engines dqm 121 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c * dqm->dev->device_info->num_sdma_queues_per_engine; dqm 124 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c void program_sh_mem_settings(struct device_queue_manager *dqm, dqm 127 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->kfd2kgd->program_sh_mem_settings( dqm 128 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kgd, qpd->vmid, dqm 137 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct kfd_dev *dev = qpd->dqm->dev; dqm 183 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct kfd_dev *dev = qpd->dqm->dev; dqm 194 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int allocate_vmid(struct device_queue_manager *dqm, dqm 200 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->vmid_bitmap == 0) dqm 203 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c bit = ffs(dqm->vmid_bitmap) - 1; dqm 204 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->vmid_bitmap &= ~(1 << bit); dqm 206 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd; dqm 211 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); dqm 212 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c program_sh_mem_settings(dqm, qpd); dqm 217 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd, dqm 223 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kfd2kgd->set_scratch_backing_va( dqm 224 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid); dqm 232 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf; dqm 247 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static void deallocate_vmid(struct device_queue_manager *dqm, dqm 251 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd; dqm 261 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c set_pasid_vmid_mapping(dqm, 0, qpd->vmid); dqm 263 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->vmid_bitmap |= (1 << bit); dqm 268 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int create_queue_nocpsch(struct device_queue_manager *dqm, dqm 277 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 279 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->total_queue_count >= max_num_of_queues_per_device) { dqm 281 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 287 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = allocate_vmid(dqm, qpd, q); dqm 302 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 305 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = allocate_hqd(dqm, q); dqm 312 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = allocate_sdma_queue(dqm, q); dqm 315 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->asic_ops.init_sdma_vm(dqm, q, qpd); dqm 323 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 325 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 348 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count++; dqm 351 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count++; dqm 353 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count++; dqm 359 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count++; dqm 361 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 370 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_hqd(dqm, q); dqm 373 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 376 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_vmid(dqm, qpd, q); dqm 378 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 382 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) dqm 389 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (pipe = dqm->next_pipe_to_allocate, i = 0; dqm 390 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c i < get_pipes_per_mec(dqm); dqm 391 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) { dqm 393 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!is_pipe_enabled(dqm, 0, pipe)) dqm 396 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->allocated_queues[pipe] != 0) { dqm 397 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c bit = ffs(dqm->allocated_queues[pipe]) - 1; dqm 398 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->allocated_queues[pipe] &= ~(1 << bit); dqm 411 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm); dqm 416 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static inline void deallocate_hqd(struct device_queue_manager *dqm, dqm 419 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->allocated_queues[q->pipe] |= (1 << q->queue); dqm 425 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, dqm 432 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 436 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_hqd(dqm, q); dqm 438 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count--; dqm 439 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 441 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count--; dqm 442 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 448 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count--; dqm 465 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev); dqm 469 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dbgdev_wave_reset_wavefronts(dqm->dev, dqm 474 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_vmid(dqm, qpd, q); dqm 478 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 483 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int destroy_queue_nocpsch(struct device_queue_manager *dqm, dqm 489 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 490 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = destroy_queue_nocpsch_locked(dqm, qpd, q); dqm 491 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 496 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int update_queue(struct device_queue_manager *dqm, struct queue *q) dqm 503 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 509 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 516 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) { dqm 517 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = unmap_queues_cpsch(dqm, dqm 545 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count++; dqm 547 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 549 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) dqm 550 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = map_queues_cpsch(dqm); dqm 565 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 569 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, dqm 577 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 593 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 604 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 608 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 612 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int evict_process_queues_cpsch(struct device_queue_manager *dqm, dqm 619 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 636 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 638 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = execute_queues_cpsch(dqm, dqm 644 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 648 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, dqm 662 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 678 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kfd2kgd->set_vm_context_page_table_base( dqm 679 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kgd, dqm 702 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 712 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count++; dqm 718 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 722 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int restore_process_queues_cpsch(struct device_queue_manager *dqm, dqm 734 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 756 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count++; dqm 758 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = execute_queues_cpsch(dqm, dqm 762 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 766 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int register_process(struct device_queue_manager *dqm, dqm 784 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 785 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c list_add(&n->list, &dqm->queues); dqm 791 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = dqm->asic_ops.update_qpd(dqm, qpd); dqm 793 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->processes_count++; dqm 795 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 800 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfd_inc_compute_active(dqm->dev); dqm 805 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unregister_process(struct device_queue_manager *dqm, dqm 815 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 817 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c list_for_each_entry_safe(cur, next, &dqm->queues, list) { dqm 821 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->processes_count--; dqm 828 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 834 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfd_dec_compute_active(dqm->dev); dqm 840 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, dqm 843 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( dqm 844 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kgd, pasid, vmid); dqm 847 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static void init_interrupts(struct device_queue_manager *dqm) dqm 851 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) dqm 852 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (is_pipe_enabled(dqm, 0, i)) dqm 853 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i); dqm 856 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int initialize_nocpsch(struct device_queue_manager *dqm) dqm 860 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); dqm 862 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm), dqm 864 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!dqm->allocated_queues) dqm 867 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mutex_init(&dqm->lock_hidden); dqm 868 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c INIT_LIST_HEAD(&dqm->queues); dqm 869 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count = dqm->next_pipe_to_allocate = 0; dqm 870 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count = 0; dqm 871 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count = 0; dqm 873 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { dqm 874 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c int pipe_offset = pipe * get_queues_per_pipe(dqm); dqm 876 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) dqm 878 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->shared_resources.queue_bitmap)) dqm 879 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->allocated_queues[pipe] |= 1 << queue; dqm 882 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1; dqm 883 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); dqm 884 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); dqm 889 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static void uninitialize(struct device_queue_manager *dqm) dqm 893 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0); dqm 895 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfree(dqm->allocated_queues); dqm 897 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfree(dqm->mqd_mgrs[i]); dqm 898 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mutex_destroy(&dqm->lock_hidden); dqm 899 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); dqm 902 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int start_nocpsch(struct device_queue_manager *dqm) dqm 904 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c init_interrupts(dqm); dqm 905 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return pm_init(&dqm->packets, dqm); dqm 908 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int stop_nocpsch(struct device_queue_manager *dqm) dqm 910 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pm_uninit(&dqm->packets); dqm 914 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int allocate_sdma_queue(struct device_queue_manager *dqm, dqm 920 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->sdma_bitmap == 0) dqm 922 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c bit = __ffs64(dqm->sdma_bitmap); dqm 923 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_bitmap &= ~(1ULL << bit); dqm 926 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c get_num_sdma_engines(dqm); dqm 928 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c get_num_sdma_engines(dqm); dqm 930 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->xgmi_sdma_bitmap == 0) dqm 932 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c bit = __ffs64(dqm->xgmi_sdma_bitmap); dqm 933 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_bitmap &= ~(1ULL << bit); dqm 941 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c q->properties.sdma_engine_id = get_num_sdma_engines(dqm) + dqm 942 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c q->sdma_id % get_num_xgmi_sdma_engines(dqm); dqm 944 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c get_num_xgmi_sdma_engines(dqm); dqm 953 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static void deallocate_sdma_queue(struct device_queue_manager *dqm, dqm 957 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (q->sdma_id >= get_num_sdma_queues(dqm)) dqm 959 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_bitmap |= (1ULL << q->sdma_id); dqm 961 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) dqm 963 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id); dqm 971 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int set_sched_resources(struct device_queue_manager *dqm) dqm 976 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap; dqm 980 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mec = (i / dqm->dev->shared_resources.num_queue_per_pipe) dqm 981 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c / dqm->dev->shared_resources.num_pipe_per_mec; dqm 983 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap)) dqm 1009 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return pm_send_set_resources(&dqm->packets, &res); dqm 1012 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int initialize_cpsch(struct device_queue_manager *dqm) dqm 1014 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); dqm 1016 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mutex_init(&dqm->lock_hidden); dqm 1017 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c INIT_LIST_HEAD(&dqm->queues); dqm 1018 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count = dqm->processes_count = 0; dqm 1019 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count = 0; dqm 1020 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count = 0; dqm 1021 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->active_runlist = false; dqm 1022 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); dqm 1023 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); dqm 1025 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); dqm 1030 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int start_cpsch(struct device_queue_manager *dqm) dqm 1036 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = pm_init(&dqm->packets, dqm); dqm 1040 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = set_sched_resources(dqm); dqm 1047 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), dqm 1048 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c &dqm->fence_mem); dqm 1053 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->fence_addr = dqm->fence_mem->cpu_ptr; dqm 1054 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; dqm 1056 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c init_interrupts(dqm); dqm 1058 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1060 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->is_hws_hang = false; dqm 1061 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0); dqm 1062 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1067 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pm_uninit(&dqm->packets); dqm 1072 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int stop_cpsch(struct device_queue_manager *dqm) dqm 1074 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1075 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0); dqm 1076 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1078 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); dqm 1079 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pm_uninit(&dqm->packets); dqm 1084 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, dqm 1088 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1089 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->total_queue_count >= max_num_of_queues_per_device) { dqm 1091 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 1092 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1100 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count++; dqm 1102 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 1105 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count++; dqm 1107 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0); dqm 1108 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1113 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, dqm 1117 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1119 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 1121 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0); dqm 1126 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count--; dqm 1128 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 1129 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1132 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, dqm 1138 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->total_queue_count >= max_num_of_queues_per_device) { dqm 1140 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 1147 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1148 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = allocate_sdma_queue(dqm, q); dqm 1149 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1158 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 1163 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->asic_ops.init_sdma_vm(dqm, q, qpd); dqm 1172 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1186 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count++; dqm 1188 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count++; dqm 1191 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count++; dqm 1192 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = execute_queues_cpsch(dqm, dqm 1200 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count++; dqm 1203 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 1205 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1213 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1214 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 1215 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1245 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unmap_sdma_queues(struct device_queue_manager *dqm) dqm 1249 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (i = 0; i < dqm->dev->device_info->num_sdma_engines + dqm 1250 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->device_info->num_xgmi_sdma_engines; i++) { dqm 1251 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, dqm 1260 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int map_queues_cpsch(struct device_queue_manager *dqm) dqm 1264 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->queue_count <= 0 || dqm->processes_count <= 0) dqm 1267 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->active_runlist) dqm 1270 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = pm_send_runlist(&dqm->packets, &dqm->queues); dqm 1276 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->active_runlist = true; dqm 1282 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int unmap_queues_cpsch(struct device_queue_manager *dqm, dqm 1288 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->is_hws_hang) dqm 1290 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!dqm->active_runlist) dqm 1294 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count, dqm->xgmi_sdma_queue_count); dqm 1296 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->sdma_queue_count > 0 || dqm->xgmi_sdma_queue_count) dqm 1297 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c unmap_sdma_queues(dqm); dqm 1299 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, dqm 1304 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c *dqm->fence_addr = KFD_FENCE_INIT; dqm 1305 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, dqm 1308 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, dqm 1313 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pm_release_ib(&dqm->packets); dqm 1314 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->active_runlist = false; dqm 1320 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int execute_queues_cpsch(struct device_queue_manager *dqm, dqm 1326 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->is_hws_hang) dqm 1328 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = unmap_queues_cpsch(dqm, filter, filter_param); dqm 1331 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->is_hws_hang = true; dqm 1332 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c schedule_work(&dqm->hw_exception_work); dqm 1336 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return map_queues_cpsch(dqm); dqm 1339 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int destroy_queue_cpsch(struct device_queue_manager *dqm, dqm 1349 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1361 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 1367 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count--; dqm 1368 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 1370 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count--; dqm 1371 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 1377 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 1378 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = execute_queues_cpsch(dqm, dqm 1388 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count--; dqm 1390 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count); dqm 1392 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1401 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1413 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static bool set_cache_memory_policy(struct device_queue_manager *dqm, dqm 1422 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!dqm->asic_ops.set_cache_memory_policy) dqm 1425 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1455 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = dqm->asic_ops.set_cache_memory_policy( dqm 1456 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm, dqm 1463 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) dqm 1464 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c program_sh_mem_settings(dqm, qpd); dqm 1471 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1475 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int set_trap_handler(struct device_queue_manager *dqm, dqm 1482 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (dqm->dev->cwsr_enabled) { dqm 1495 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int process_termination_nocpsch(struct device_queue_manager *dqm, dqm 1503 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1509 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c ret = destroy_queue_nocpsch_locked(dqm, qpd, q); dqm 1515 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { dqm 1519 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->processes_count--; dqm 1525 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1531 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfd_dec_compute_active(dqm->dev); dqm 1536 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int get_wave_state(struct device_queue_manager *dqm, dqm 1545 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1553 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_COMPUTE]; dqm 1564 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1568 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int process_termination_cpsch(struct device_queue_manager *dqm, dqm 1582 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1587 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 1589 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count--; dqm 1596 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sdma_queue_count--; dqm 1597 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 1599 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->xgmi_sdma_queue_count--; dqm 1600 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_sdma_queue(dqm, q); dqm 1604 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->queue_count--; dqm 1606 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->total_queue_count--; dqm 1610 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) { dqm 1614 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->processes_count--; dqm 1620 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c retval = execute_queues_cpsch(dqm, filter, 0); dqm 1621 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) { dqm 1622 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev); dqm 1623 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process); dqm 1627 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 1633 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfd_dec_compute_active(dqm->dev); dqm 1639 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( dqm 1649 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int init_mqd_managers(struct device_queue_manager *dqm) dqm 1655 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev); dqm 1660 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->mqd_mgrs[i] = mqd_mgr; dqm 1667 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfree(dqm->mqd_mgrs[j]); dqm 1668 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->mqd_mgrs[j] = NULL; dqm 1675 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) dqm 1678 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct kfd_dev *dev = dqm->dev; dqm 1679 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd; dqm 1680 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size * dqm 1684 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size; dqm 1695 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct device_queue_manager *dqm; dqm 1699 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm = kzalloc(sizeof(*dqm), GFP_KERNEL); dqm 1700 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!dqm) dqm 1712 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS; dqm 1715 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->sched_policy = sched_policy; dqm 1719 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev = dev; dqm 1720 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c switch (dqm->sched_policy) { dqm 1724 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.create_queue = create_queue_cpsch; dqm 1725 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.initialize = initialize_cpsch; dqm 1726 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.start = start_cpsch; dqm 1727 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.stop = stop_cpsch; dqm 1728 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.destroy_queue = destroy_queue_cpsch; dqm 1729 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.update_queue = update_queue; dqm 1730 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.register_process = register_process; dqm 1731 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.unregister_process = unregister_process; dqm 1732 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.uninitialize = uninitialize; dqm 1733 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; dqm 1734 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; dqm 1735 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.set_cache_memory_policy = set_cache_memory_policy; dqm 1736 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.set_trap_handler = set_trap_handler; dqm 1737 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.process_termination = process_termination_cpsch; dqm 1738 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.evict_process_queues = evict_process_queues_cpsch; dqm 1739 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.restore_process_queues = restore_process_queues_cpsch; dqm 1740 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.get_wave_state = get_wave_state; dqm 1744 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.start = start_nocpsch; dqm 1745 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.stop = stop_nocpsch; dqm 1746 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.create_queue = create_queue_nocpsch; dqm 1747 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.destroy_queue = destroy_queue_nocpsch; dqm 1748 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.update_queue = update_queue; dqm 1749 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.register_process = register_process; dqm 1750 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.unregister_process = unregister_process; dqm 1751 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.initialize = initialize_nocpsch; dqm 1752 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.uninitialize = uninitialize; dqm 1753 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.set_cache_memory_policy = set_cache_memory_policy; dqm 1754 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.set_trap_handler = set_trap_handler; dqm 1755 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.process_termination = process_termination_nocpsch; dqm 1756 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.evict_process_queues = evict_process_queues_nocpsch; dqm 1757 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.restore_process_queues = dqm 1759 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.get_wave_state = get_wave_state; dqm 1762 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pr_err("Invalid scheduling policy %d\n", dqm->sched_policy); dqm 1768 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c device_queue_manager_init_vi(&dqm->asic_ops); dqm 1772 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c device_queue_manager_init_cik(&dqm->asic_ops); dqm 1776 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c device_queue_manager_init_cik_hawaii(&dqm->asic_ops); dqm 1785 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c device_queue_manager_init_vi_tonga(&dqm->asic_ops); dqm 1793 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c device_queue_manager_init_v9(&dqm->asic_ops); dqm 1796 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c device_queue_manager_init_v10_navi10(&dqm->asic_ops); dqm 1804 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (init_mqd_managers(dqm)) dqm 1807 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (allocate_hiq_sdma_mqd(dqm)) { dqm 1812 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c if (!dqm->ops.initialize(dqm)) dqm 1813 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c return dqm; dqm 1816 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfree(dqm); dqm 1828 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c void device_queue_manager_uninit(struct device_queue_manager *dqm) dqm 1830 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->ops.uninitialize(dqm); dqm 1831 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd); dqm 1832 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c kfree(dqm); dqm 1835 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c int kfd_process_vm_fault(struct device_queue_manager *dqm, dqm 1844 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c pdd = kfd_get_process_device_data(dqm->dev, p); dqm 1846 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd); dqm 1854 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct device_queue_manager *dqm = container_of(work, dqm 1856 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c amdgpu_amdkfd_gpu_reset(dqm->dev->kgd); dqm 1884 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c struct device_queue_manager *dqm = data; dqm 1889 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd, dqm 1894 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1, dqm 1895 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm), dqm 1902 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { dqm 1903 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c int pipe_offset = pipe * get_queues_per_pipe(dqm); dqm 1905 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) { dqm 1907 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->shared_resources.queue_bitmap)) dqm 1910 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c r = dqm->dev->kfd2kgd->hqd_dump( dqm 1911 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kgd, pipe, queue, &dump, &n_regs); dqm 1923 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) { dqm 1925 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c queue < dqm->dev->device_info->num_sdma_queues_per_engine; dqm 1927 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c r = dqm->dev->kfd2kgd->hqd_sdma_dump( dqm 1928 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->dev->kgd, pipe, queue, &dump, &n_regs); dqm 1943 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c int dqm_debugfs_execute_queues(struct device_queue_manager *dqm) dqm 1947 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_lock(dqm); dqm 1948 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm->active_runlist = true; dqm 1949 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0); dqm 1950 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c dqm_unlock(dqm); dqm 85 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*create_queue)(struct device_queue_manager *dqm, dqm 89 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*destroy_queue)(struct device_queue_manager *dqm, dqm 93 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*update_queue)(struct device_queue_manager *dqm, dqm 96 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*register_process)(struct device_queue_manager *dqm, dqm 99 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*unregister_process)(struct device_queue_manager *dqm, dqm 102 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*initialize)(struct device_queue_manager *dqm); dqm 103 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*start)(struct device_queue_manager *dqm); dqm 104 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*stop)(struct device_queue_manager *dqm); dqm 105 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h void (*uninitialize)(struct device_queue_manager *dqm); dqm 106 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*create_kernel_queue)(struct device_queue_manager *dqm, dqm 110 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h void (*destroy_kernel_queue)(struct device_queue_manager *dqm, dqm 114 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, dqm 121 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*set_trap_handler)(struct device_queue_manager *dqm, dqm 126 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*process_termination)(struct device_queue_manager *dqm, dqm 129 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*evict_process_queues)(struct device_queue_manager *dqm, dqm 131 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*restore_process_queues)(struct device_queue_manager *dqm, dqm 134 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*get_wave_state)(struct device_queue_manager *dqm, dqm 142 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h int (*update_qpd)(struct device_queue_manager *dqm, dqm 144 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, dqm 150 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h void (*init_sdma_vm)(struct device_queue_manager *dqm, dqm 215 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h void program_sh_mem_settings(struct device_queue_manager *dqm, dqm 217 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h unsigned int get_queues_num(struct device_queue_manager *dqm); dqm 218 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); dqm 219 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); dqm 220 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); dqm 221 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); dqm 238 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h static inline void dqm_lock(struct device_queue_manager *dqm) dqm 240 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h mutex_lock(&dqm->lock_hidden); dqm 241 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h dqm->saved_flags = memalloc_nofs_save(); dqm 243 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h static inline void dqm_unlock(struct device_queue_manager *dqm) dqm 245 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h memalloc_nofs_restore(dqm->saved_flags); dqm 246 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h mutex_unlock(&dqm->lock_hidden); dqm 29 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm, dqm 35 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static int update_qpd_cik(struct device_queue_manager *dqm, dqm 37 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static int update_qpd_cik_hawaii(struct device_queue_manager *dqm, dqm 39 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, dqm 41 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static void init_sdma_vm_hawaii(struct device_queue_manager *dqm, dqm 90 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm, dqm 116 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static int update_qpd_cik(struct device_queue_manager *dqm, dqm 150 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static int update_qpd_cik_hawaii(struct device_queue_manager *dqm, dqm 180 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, dqm 196 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c static void init_sdma_vm_hawaii(struct device_queue_manager *dqm, dqm 29 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c static int update_qpd_v10(struct device_queue_manager *dqm, dqm 31 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c static void init_sdma_vm_v10(struct device_queue_manager *dqm, struct queue *q, dqm 51 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c static int update_qpd_v10(struct device_queue_manager *dqm, dqm 83 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v10.c static void init_sdma_vm_v10(struct device_queue_manager *dqm, struct queue *q, dqm 30 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c static int update_qpd_v9(struct device_queue_manager *dqm, dqm 32 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c static void init_sdma_vm_v9(struct device_queue_manager *dqm, struct queue *q, dqm 52 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c static int update_qpd_v9(struct device_queue_manager *dqm, dqm 65 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c !dqm->dev->device_info->needs_iommu_device) dqm 80 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c static void init_sdma_vm_v9(struct device_queue_manager *dqm, struct queue *q, dqm 29 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, dqm 35 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static bool set_cache_memory_policy_vi_tonga(struct device_queue_manager *dqm, dqm 41 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static int update_qpd_vi(struct device_queue_manager *dqm, dqm 43 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static int update_qpd_vi_tonga(struct device_queue_manager *dqm, dqm 45 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, dqm 47 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static void init_sdma_vm_tonga(struct device_queue_manager *dqm, dqm 97 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, dqm 126 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static bool set_cache_memory_policy_vi_tonga(struct device_queue_manager *dqm, dqm 153 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static int update_qpd_vi(struct device_queue_manager *dqm, dqm 194 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static int update_qpd_vi_tonga(struct device_queue_manager *dqm, dqm 228 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, dqm 244 drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c static void init_sdma_vm_tonga(struct device_queue_manager *dqm, dqm 50 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c if (!pasid && dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { dqm 121 drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c kfd_process_vm_fault(dev->dqm, pasid); dqm 61 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_DIQ]; dqm 64 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; dqm 119 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v10.c struct kfd_dev *kfd = pm->dqm->dev; dqm 130 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v10.c concurrent_proc_cnt = min(pm->dqm->processes_count, dqm 115 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c struct kfd_dev *kfd = pm->dqm->dev; dqm 126 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c concurrent_proc_cnt = min(pm->dqm->processes_count, dqm 120 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c struct kfd_dev *kfd = pm->dqm->dev; dqm 134 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c concurrent_proc_cnt = min(pm->dqm->processes_count, dqm 56 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c mqd_mem_obj->gtt_mem = dev->dqm->hiq_sdma_mqd.gtt_mem; dqm 57 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr; dqm 58 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c mqd_mem_obj->cpu_ptr = dev->dqm->hiq_sdma_mqd.cpu_ptr; dqm 76 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c dev->dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size; dqm 78 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c offset += dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size; dqm 80 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c mqd_mem_obj->gtt_mem = (void *)((uint64_t)dev->dqm->hiq_sdma_mqd.gtt_mem dqm 82 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr + offset; dqm 84 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c dev->dqm->hiq_sdma_mqd.cpu_ptr + offset); dqm 47 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c struct kfd_dev *dev = pm->dqm->dev; dqm 49 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c process_count = pm->dqm->processes_count; dqm 50 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c queue_count = pm->dqm->queue_count; dqm 51 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c compute_queue_count = queue_count - pm->dqm->sdma_queue_count - dqm 52 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c pm->dqm->xgmi_sdma_queue_count; dqm 65 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c compute_queue_count > get_queues_num(pm->dqm)) { dqm 100 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c retval = kfd_gtt_sa_allocate(pm->dqm->dev, *rl_buffer_size, dqm 144 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c pm->dqm->processes_count, pm->dqm->queue_count); dqm 150 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c if (proccesses_mapped >= pm->dqm->processes_count) { dqm 223 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm) dqm 225 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c switch (dqm->dev->device_info->asic_family) { dqm 250 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c dqm->dev->device_info->asic_family); dqm 254 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c pm->dqm = dqm; dqm 256 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ); dqm 407 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c kfd_gtt_sa_free(pm->dqm->dev, pm->ib_buffer_obj); dqm 271 drivers/gpu/drm/amd/amdkfd/kfd_priv.h struct device_queue_manager *dqm; dqm 545 drivers/gpu/drm/amd/amdkfd/kfd_priv.h struct device_queue_manager *dqm; dqm 885 drivers/gpu/drm/amd/amdkfd/kfd_priv.h void device_queue_manager_uninit(struct device_queue_manager *dqm); dqm 889 drivers/gpu/drm/amd/amdkfd/kfd_priv.h int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid); dqm 932 drivers/gpu/drm/amd/amdkfd/kfd_priv.h struct device_queue_manager *dqm; dqm 976 drivers/gpu/drm/amd/amdkfd/kfd_priv.h int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); dqm 1057 drivers/gpu/drm/amd/amdkfd/kfd_priv.h int dqm_debugfs_execute_queues(struct device_queue_manager *dqm); dqm 745 drivers/gpu/drm/amd/amdkfd/kfd_process.c pdd->qpd.dqm = dev->dqm; dqm 953 drivers/gpu/drm/amd/amdkfd/kfd_process.c r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, dqm 971 drivers/gpu/drm/amd/amdkfd/kfd_process.c if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, dqm 988 drivers/gpu/drm/amd/amdkfd/kfd_process.c r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, dqm 1152 drivers/gpu/drm/amd/amdkfd/kfd_process.c if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { dqm 74 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd); dqm 123 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, dqm 232 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dev->dqm->ops.register_process(dev->dqm, &pdd->qpd); dqm 243 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c if ((type == KFD_QUEUE_TYPE_SDMA && dev->dqm->sdma_queue_count dqm 244 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c >= get_num_sdma_queues(dev->dqm)) || dqm 246 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dev->dqm->xgmi_sdma_queue_count dqm 247 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c >= get_num_xgmi_sdma_queues(dev->dqm))) { dqm 258 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd); dqm 265 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c if ((dev->dqm->sched_policy == dqm 267 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) || dqm 268 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c (dev->dqm->queue_count >= get_queues_num(dev->dqm)))) { dqm 279 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd); dqm 292 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c retval = dev->dqm->ops.create_kernel_queue(dev->dqm, dqm 333 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd); dqm 341 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c struct device_queue_manager *dqm; dqm 345 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dqm = NULL; dqm 371 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dqm = pqn->kq->dev->dqm; dqm 372 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd); dqm 377 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dqm = pqn->q->device->dqm; dqm 378 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q); dqm 404 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c dqm->ops.unregister_process(dqm, &pdd->qpd); dqm 427 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, dqm 455 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, dqm 491 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm, dqm 530 drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type]; dqm 1291 drivers/gpu/drm/amd/amdkfd/kfd_topology.c dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? dqm 1479 drivers/gpu/drm/amd/amdkfd/kfd_topology.c r = dqm_debugfs_hqds(m, dev->gpu->dqm); dqm 1504 drivers/gpu/drm/amd/amdkfd/kfd_topology.c r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets);