51 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
55 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
57 static uint64_t get_vmem_size(struct kgd_dev *kgd);
58 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
60 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
61 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
67 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
71 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
74 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
76 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
77 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
79 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
80 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
83 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
86 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
87 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
89 static int kgd_address_watch_disable(struct kgd_dev *kgd);
90 static int kgd_address_watch_execute(struct kgd_dev *kgd,
95 static int kgd_wave_control_execute(struct kgd_dev *kgd,
98 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
102 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid);
103 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
105 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
230 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,  in alloc_gtt_mem()  argument
234 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in alloc_gtt_mem()
238 	BUG_ON(kgd == NULL);  in alloc_gtt_mem()
291 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)  in free_gtt_mem()  argument
305 static uint64_t get_vmem_size(struct kgd_dev *kgd)  in get_vmem_size()  argument
307 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in get_vmem_size()
309 	BUG_ON(kgd == NULL);  in get_vmem_size()
314 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)  in get_gpu_clock_counter()  argument
316 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in get_gpu_clock_counter()
321 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)  in get_max_engine_clock_in_mhz()  argument
323 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in get_max_engine_clock_in_mhz()
329 static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)  in get_radeon_device()  argument
331 	return (struct radeon_device *)kgd;  in get_radeon_device()
334 static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)  in write_register()  argument
336 	struct radeon_device *rdev = get_radeon_device(kgd);  in write_register()
341 static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)  in read_register()  argument
343 	struct radeon_device *rdev = get_radeon_device(kgd);  in read_register()
348 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,  in lock_srbm()  argument
351 	struct radeon_device *rdev = get_radeon_device(kgd);  in lock_srbm()
355 	write_register(kgd, SRBM_GFX_CNTL, value);  in lock_srbm()
358 static void unlock_srbm(struct kgd_dev *kgd)  in unlock_srbm()  argument
360 	struct radeon_device *rdev = get_radeon_device(kgd);  in unlock_srbm()
362 	write_register(kgd, SRBM_GFX_CNTL, 0);  in unlock_srbm()
366 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,  in acquire_queue()  argument
372 	lock_srbm(kgd, mec, pipe, queue_id, 0);  in acquire_queue()
375 static void release_queue(struct kgd_dev *kgd)  in release_queue()  argument
377 	unlock_srbm(kgd);  in release_queue()
380 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,  in kgd_program_sh_mem_settings()  argument
386 	lock_srbm(kgd, 0, 0, 0, vmid);  in kgd_program_sh_mem_settings()
388 	write_register(kgd, SH_MEM_CONFIG, sh_mem_config);  in kgd_program_sh_mem_settings()
389 	write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);  in kgd_program_sh_mem_settings()
390 	write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);  in kgd_program_sh_mem_settings()
391 	write_register(kgd, SH_MEM_BASES, sh_mem_bases);  in kgd_program_sh_mem_settings()
393 	unlock_srbm(kgd);  in kgd_program_sh_mem_settings()
396 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,  in kgd_set_pasid_vmid_mapping()  argument
409 	write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),  in kgd_set_pasid_vmid_mapping()
412 	while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &  in kgd_set_pasid_vmid_mapping()
415 	write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);  in kgd_set_pasid_vmid_mapping()
418 	write_register(kgd, IH_VMID_0_LUT + vmid * sizeof(uint32_t),  in kgd_set_pasid_vmid_mapping()
424 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,  in kgd_init_pipeline()  argument
430 	lock_srbm(kgd, mec, pipe, 0, 0);  in kgd_init_pipeline()
431 	write_register(kgd, CP_HPD_EOP_BASE_ADDR,  in kgd_init_pipeline()
433 	write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,  in kgd_init_pipeline()
435 	write_register(kgd, CP_HPD_EOP_VMID, 0);  in kgd_init_pipeline()
436 	write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);  in kgd_init_pipeline()
437 	unlock_srbm(kgd);  in kgd_init_pipeline()
442 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)  in kgd_init_interrupts()  argument
450 	lock_srbm(kgd, mec, pipe, 0, 0);  in kgd_init_interrupts()
452 	write_register(kgd, CPC_INT_CNTL,  in kgd_init_interrupts()
455 	unlock_srbm(kgd);  in kgd_init_interrupts()
482 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,  in kgd_hqd_load()  argument
492 	acquire_queue(kgd, pipe_id, queue_id);  in kgd_hqd_load()
493 	write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);  in kgd_hqd_load()
494 	write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);  in kgd_hqd_load()
495 	write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);  in kgd_hqd_load()
497 	write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);  in kgd_hqd_load()
498 	write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);  in kgd_hqd_load()
499 	write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);  in kgd_hqd_load()
501 	write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);  in kgd_hqd_load()
502 	write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);  in kgd_hqd_load()
503 	write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);  in kgd_hqd_load()
505 	write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);  in kgd_hqd_load()
507 	write_register(kgd, CP_HQD_PERSISTENT_STATE,  in kgd_hqd_load()
509 	write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);  in kgd_hqd_load()
510 	write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);  in kgd_hqd_load()
512 	write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,  in kgd_hqd_load()
515 	write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,  in kgd_hqd_load()
518 	write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,  in kgd_hqd_load()
521 	write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,  in kgd_hqd_load()
524 	write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,  in kgd_hqd_load()
527 	write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,  in kgd_hqd_load()
530 	write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);  in kgd_hqd_load()
532 	write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,  in kgd_hqd_load()
535 	write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,  in kgd_hqd_load()
538 	write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,  in kgd_hqd_load()
541 	write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);  in kgd_hqd_load()
543 	write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);  in kgd_hqd_load()
545 	write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);  in kgd_hqd_load()
546 	write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);  in kgd_hqd_load()
548 	write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);  in kgd_hqd_load()
551 		write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);  in kgd_hqd_load()
553 	write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);  in kgd_hqd_load()
554 	release_queue(kgd);  in kgd_hqd_load()
559 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)  in kgd_hqd_sdma_load()  argument
567 	write_register(kgd,  in kgd_hqd_sdma_load()
571 	write_register(kgd,  in kgd_hqd_sdma_load()
575 	write_register(kgd,  in kgd_hqd_sdma_load()
579 	write_register(kgd,  in kgd_hqd_sdma_load()
583 	write_register(kgd,  in kgd_hqd_sdma_load()
587 	write_register(kgd,  in kgd_hqd_sdma_load()
591 	write_register(kgd,  in kgd_hqd_sdma_load()
598 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,  in kgd_hqd_is_occupied()  argument
605 	acquire_queue(kgd, pipe_id, queue_id);  in kgd_hqd_is_occupied()
606 	act = read_register(kgd, CP_HQD_ACTIVE);  in kgd_hqd_is_occupied()
611 		if (low == read_register(kgd, CP_HQD_PQ_BASE) &&  in kgd_hqd_is_occupied()
612 				high == read_register(kgd, CP_HQD_PQ_BASE_HI))  in kgd_hqd_is_occupied()
615 	release_queue(kgd);  in kgd_hqd_is_occupied()
619 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)  in kgd_hqd_sdma_is_occupied()  argument
628 	sdma_rlc_rb_cntl = read_register(kgd,  in kgd_hqd_sdma_is_occupied()
637 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,  in kgd_hqd_destroy()  argument
643 	acquire_queue(kgd, pipe_id, queue_id);  in kgd_hqd_destroy()
644 	write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);  in kgd_hqd_destroy()
646 	write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);  in kgd_hqd_destroy()
649 		temp = read_register(kgd, CP_HQD_ACTIVE);  in kgd_hqd_destroy()
655 			release_queue(kgd);  in kgd_hqd_destroy()
662 	release_queue(kgd);  in kgd_hqd_destroy()
666 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,  in kgd_hqd_sdma_destroy()  argument
676 	temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);  in kgd_hqd_sdma_destroy()
678 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);  in kgd_hqd_sdma_destroy()
681 		temp = read_register(kgd, sdma_base_addr +  in kgd_hqd_sdma_destroy()
691 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);  in kgd_hqd_sdma_destroy()
692 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);  in kgd_hqd_sdma_destroy()
693 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);  in kgd_hqd_sdma_destroy()
694 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);  in kgd_hqd_sdma_destroy()
699 static int kgd_address_watch_disable(struct kgd_dev *kgd)  in kgd_address_watch_disable()  argument
712 		write_register(kgd,  in kgd_address_watch_disable()
720 static int kgd_address_watch_execute(struct kgd_dev *kgd,  in kgd_address_watch_execute()  argument
732 	write_register(kgd,  in kgd_address_watch_execute()
737 	write_register(kgd,  in kgd_address_watch_execute()
742 	write_register(kgd,  in kgd_address_watch_execute()
750 	write_register(kgd,  in kgd_address_watch_execute()
758 static int kgd_wave_control_execute(struct kgd_dev *kgd,  in kgd_wave_control_execute()  argument
762 	struct radeon_device *rdev = get_radeon_device(kgd);  in kgd_wave_control_execute()
767 	write_register(kgd, GRBM_GFX_INDEX, gfx_index_val);  in kgd_wave_control_execute()
768 	write_register(kgd, SQ_CMD, sq_cmd);  in kgd_wave_control_execute()
775 	write_register(kgd, GRBM_GFX_INDEX, data);  in kgd_wave_control_execute()
782 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,  in kgd_address_watch_get_offset()  argument
789 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid)  in get_atc_vmid_pasid_mapping_valid()  argument
792 	struct radeon_device *rdev = (struct radeon_device *) kgd;  in get_atc_vmid_pasid_mapping_valid()
798 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,  in get_atc_vmid_pasid_mapping_pasid()  argument
802 	struct radeon_device *rdev = (struct radeon_device *) kgd;  in get_atc_vmid_pasid_mapping_pasid()
808 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)  in write_vmid_invalidate_request()  argument
810 	struct radeon_device *rdev = (struct radeon_device *) kgd;  in write_vmid_invalidate_request()
815 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)  in get_fw_version()  argument
817 	struct radeon_device *rdev = (struct radeon_device *) kgd;  in get_fw_version()
820 	BUG_ON(kgd == NULL || rdev->mec_fw == NULL);  in get_fw_version()