1/* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24#include <linux/slab.h> 25#include <linux/list.h> 26#include <linux/types.h> 27#include <linux/printk.h> 28#include <linux/bitops.h> 29#include <linux/sched.h> 30#include "kfd_priv.h" 31#include "kfd_device_queue_manager.h" 32#include "kfd_mqd_manager.h" 33#include "cik_regs.h" 34#include "kfd_kernel_queue.h" 35 36/* Size of the per-pipe EOP queue */ 37#define CIK_HPD_EOP_BYTES_LOG2 11 38#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2) 39 40static int set_pasid_vmid_mapping(struct device_queue_manager *dqm, 41 unsigned int pasid, unsigned int vmid); 42 43static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 44 struct queue *q, 45 struct qcm_process_device *qpd); 46 47static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock); 48static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock); 49 50static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 51 struct queue *q, 52 struct qcm_process_device *qpd); 53 54static void deallocate_sdma_queue(struct device_queue_manager *dqm, 55 unsigned int sdma_queue_id); 56 57static inline 58enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) 59{ 60 if (type == KFD_QUEUE_TYPE_SDMA) 61 return KFD_MQD_TYPE_SDMA; 62 return KFD_MQD_TYPE_CP; 63} 64 65unsigned int get_first_pipe(struct device_queue_manager *dqm) 66{ 67 BUG_ON(!dqm || !dqm->dev); 68 return dqm->dev->shared_resources.first_compute_pipe; 69} 70 71unsigned int get_pipes_num(struct device_queue_manager *dqm) 72{ 73 BUG_ON(!dqm || !dqm->dev); 74 return dqm->dev->shared_resources.compute_pipe_count; 75} 76 77static inline unsigned int get_pipes_num_cpsch(void) 78{ 79 return PIPE_PER_ME_CP_SCHEDULING; 80} 81 82void program_sh_mem_settings(struct device_queue_manager *dqm, 83 struct qcm_process_device *qpd) 84{ 85 return dqm->dev->kfd2kgd->program_sh_mem_settings( 86 dqm->dev->kgd, qpd->vmid, 87 qpd->sh_mem_config, 88 qpd->sh_mem_ape1_base, 89 qpd->sh_mem_ape1_limit, 90 qpd->sh_mem_bases); 91} 92 93static int allocate_vmid(struct device_queue_manager *dqm, 94 struct qcm_process_device *qpd, 95 struct queue *q) 96{ 97 int bit, allocated_vmid; 98 99 if (dqm->vmid_bitmap == 0) 100 return -ENOMEM; 101 102 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM); 103 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 104 105 /* Kaveri kfd vmid's starts from vmid 8 */ 106 allocated_vmid = bit + KFD_VMID_START_OFFSET; 107 pr_debug("kfd: vmid allocation %d\n", allocated_vmid); 108 qpd->vmid = allocated_vmid; 109 q->properties.vmid = allocated_vmid; 110 111 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid); 112 program_sh_mem_settings(dqm, qpd); 113 114 return 0; 115} 116 117static void deallocate_vmid(struct device_queue_manager *dqm, 118 struct qcm_process_device *qpd, 119 struct queue *q) 120{ 121 int bit = qpd->vmid - KFD_VMID_START_OFFSET; 122 123 /* Release the vmid mapping */ 124 set_pasid_vmid_mapping(dqm, 0, qpd->vmid); 125 126 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap); 127 qpd->vmid = 0; 128 q->properties.vmid = 0; 129} 130 131static int create_queue_nocpsch(struct device_queue_manager *dqm, 132 struct queue *q, 133 struct qcm_process_device *qpd, 134 int *allocated_vmid) 135{ 136 int retval; 137 138 BUG_ON(!dqm || !q || !qpd || !allocated_vmid); 139 140 pr_debug("kfd: In func %s\n", __func__); 141 print_queue(q); 142 143 mutex_lock(&dqm->lock); 144 145 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 146 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 147 dqm->total_queue_count); 148 mutex_unlock(&dqm->lock); 149 return -EPERM; 150 } 151 152 if (list_empty(&qpd->queues_list)) { 153 retval = allocate_vmid(dqm, qpd, q); 154 if (retval != 0) { 155 mutex_unlock(&dqm->lock); 156 return retval; 157 } 158 } 159 *allocated_vmid = qpd->vmid; 160 q->properties.vmid = qpd->vmid; 161 162 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) 163 retval = create_compute_queue_nocpsch(dqm, q, qpd); 164 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 165 retval = create_sdma_queue_nocpsch(dqm, q, qpd); 166 167 if (retval != 0) { 168 if (list_empty(&qpd->queues_list)) { 169 deallocate_vmid(dqm, qpd, q); 170 *allocated_vmid = 0; 171 } 172 mutex_unlock(&dqm->lock); 173 return retval; 174 } 175 176 list_add(&q->list, &qpd->queues_list); 177 if (q->properties.is_active) 178 dqm->queue_count++; 179 180 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 181 dqm->sdma_queue_count++; 182 183 /* 184 * Unconditionally increment this counter, regardless of the queue's 185 * type or whether the queue is active. 186 */ 187 dqm->total_queue_count++; 188 pr_debug("Total of %d queues are accountable so far\n", 189 dqm->total_queue_count); 190 191 mutex_unlock(&dqm->lock); 192 return 0; 193} 194 195static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q) 196{ 197 bool set; 198 int pipe, bit, i; 199 200 set = false; 201 202 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm); 203 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) { 204 if (dqm->allocated_queues[pipe] != 0) { 205 bit = find_first_bit( 206 (unsigned long *)&dqm->allocated_queues[pipe], 207 QUEUES_PER_PIPE); 208 209 clear_bit(bit, 210 (unsigned long *)&dqm->allocated_queues[pipe]); 211 q->pipe = pipe; 212 q->queue = bit; 213 set = true; 214 break; 215 } 216 } 217 218 if (set == false) 219 return -EBUSY; 220 221 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n", 222 __func__, q->pipe, q->queue); 223 /* horizontal hqd allocation */ 224 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm); 225 226 return 0; 227} 228 229static inline void deallocate_hqd(struct device_queue_manager *dqm, 230 struct queue *q) 231{ 232 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]); 233} 234 235static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, 236 struct queue *q, 237 struct qcm_process_device *qpd) 238{ 239 int retval; 240 struct mqd_manager *mqd; 241 242 BUG_ON(!dqm || !q || !qpd); 243 244 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 245 if (mqd == NULL) 246 return -ENOMEM; 247 248 retval = allocate_hqd(dqm, q); 249 if (retval != 0) 250 return retval; 251 252 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 253 &q->gart_mqd_addr, &q->properties); 254 if (retval != 0) { 255 deallocate_hqd(dqm, q); 256 return retval; 257 } 258 259 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n", 260 q->pipe, 261 q->queue); 262 263 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, 264 q->queue, (uint32_t __user *) q->properties.write_ptr); 265 if (retval != 0) { 266 deallocate_hqd(dqm, q); 267 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 268 return retval; 269 } 270 271 return 0; 272} 273 274static int destroy_queue_nocpsch(struct device_queue_manager *dqm, 275 struct qcm_process_device *qpd, 276 struct queue *q) 277{ 278 int retval; 279 struct mqd_manager *mqd; 280 281 BUG_ON(!dqm || !q || !q->mqd || !qpd); 282 283 retval = 0; 284 285 pr_debug("kfd: In Func %s\n", __func__); 286 287 mutex_lock(&dqm->lock); 288 289 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { 290 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 291 if (mqd == NULL) { 292 retval = -ENOMEM; 293 goto out; 294 } 295 deallocate_hqd(dqm, q); 296 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { 297 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 298 if (mqd == NULL) { 299 retval = -ENOMEM; 300 goto out; 301 } 302 dqm->sdma_queue_count--; 303 deallocate_sdma_queue(dqm, q->sdma_id); 304 } else { 305 pr_debug("q->properties.type is invalid (%d)\n", 306 q->properties.type); 307 retval = -EINVAL; 308 goto out; 309 } 310 311 retval = mqd->destroy_mqd(mqd, q->mqd, 312 KFD_PREEMPT_TYPE_WAVEFRONT_RESET, 313 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS, 314 q->pipe, q->queue); 315 316 if (retval != 0) 317 goto out; 318 319 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 320 321 list_del(&q->list); 322 if (list_empty(&qpd->queues_list)) 323 deallocate_vmid(dqm, qpd, q); 324 if (q->properties.is_active) 325 dqm->queue_count--; 326 327 /* 328 * Unconditionally decrement this counter, regardless of the queue's 329 * type 330 */ 331 dqm->total_queue_count--; 332 pr_debug("Total of %d queues are accountable so far\n", 333 dqm->total_queue_count); 334 335out: 336 mutex_unlock(&dqm->lock); 337 return retval; 338} 339 340static int update_queue(struct device_queue_manager *dqm, struct queue *q) 341{ 342 int retval; 343 struct mqd_manager *mqd; 344 bool prev_active = false; 345 346 BUG_ON(!dqm || !q || !q->mqd); 347 348 mutex_lock(&dqm->lock); 349 mqd = dqm->ops.get_mqd_manager(dqm, 350 get_mqd_type_from_queue_type(q->properties.type)); 351 if (mqd == NULL) { 352 mutex_unlock(&dqm->lock); 353 return -ENOMEM; 354 } 355 356 if (q->properties.is_active == true) 357 prev_active = true; 358 359 /* 360 * 361 * check active state vs. the previous state 362 * and modify counter accordingly 363 */ 364 retval = mqd->update_mqd(mqd, q->mqd, &q->properties); 365 if ((q->properties.is_active == true) && (prev_active == false)) 366 dqm->queue_count++; 367 else if ((q->properties.is_active == false) && (prev_active == true)) 368 dqm->queue_count--; 369 370 if (sched_policy != KFD_SCHED_POLICY_NO_HWS) 371 retval = execute_queues_cpsch(dqm, false); 372 373 mutex_unlock(&dqm->lock); 374 return retval; 375} 376 377static struct mqd_manager *get_mqd_manager_nocpsch( 378 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type) 379{ 380 struct mqd_manager *mqd; 381 382 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX); 383 384 pr_debug("kfd: In func %s mqd type %d\n", __func__, type); 385 386 mqd = dqm->mqds[type]; 387 if (!mqd) { 388 mqd = mqd_manager_init(type, dqm->dev); 389 if (mqd == NULL) 390 pr_err("kfd: mqd manager is NULL"); 391 dqm->mqds[type] = mqd; 392 } 393 394 return mqd; 395} 396 397static int register_process_nocpsch(struct device_queue_manager *dqm, 398 struct qcm_process_device *qpd) 399{ 400 struct device_process_node *n; 401 int retval; 402 403 BUG_ON(!dqm || !qpd); 404 405 pr_debug("kfd: In func %s\n", __func__); 406 407 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL); 408 if (!n) 409 return -ENOMEM; 410 411 n->qpd = qpd; 412 413 mutex_lock(&dqm->lock); 414 list_add(&n->list, &dqm->queues); 415 416 retval = dqm->ops_asic_specific.register_process(dqm, qpd); 417 418 dqm->processes_count++; 419 420 mutex_unlock(&dqm->lock); 421 422 return retval; 423} 424 425static int unregister_process_nocpsch(struct device_queue_manager *dqm, 426 struct qcm_process_device *qpd) 427{ 428 int retval; 429 struct device_process_node *cur, *next; 430 431 BUG_ON(!dqm || !qpd); 432 433 pr_debug("In func %s\n", __func__); 434 435 pr_debug("qpd->queues_list is %s\n", 436 list_empty(&qpd->queues_list) ? "empty" : "not empty"); 437 438 retval = 0; 439 mutex_lock(&dqm->lock); 440 441 list_for_each_entry_safe(cur, next, &dqm->queues, list) { 442 if (qpd == cur->qpd) { 443 list_del(&cur->list); 444 kfree(cur); 445 dqm->processes_count--; 446 goto out; 447 } 448 } 449 /* qpd not found in dqm list */ 450 retval = 1; 451out: 452 mutex_unlock(&dqm->lock); 453 return retval; 454} 455 456static int 457set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid, 458 unsigned int vmid) 459{ 460 uint32_t pasid_mapping; 461 462 pasid_mapping = (pasid == 0) ? 0 : 463 (uint32_t)pasid | 464 ATC_VMID_PASID_MAPPING_VALID; 465 466 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping( 467 dqm->dev->kgd, pasid_mapping, 468 vmid); 469} 470 471int init_pipelines(struct device_queue_manager *dqm, 472 unsigned int pipes_num, unsigned int first_pipe) 473{ 474 void *hpdptr; 475 struct mqd_manager *mqd; 476 unsigned int i, err, inx; 477 uint64_t pipe_hpd_addr; 478 479 BUG_ON(!dqm || !dqm->dev); 480 481 pr_debug("kfd: In func %s\n", __func__); 482 483 /* 484 * Allocate memory for the HPDs. This is hardware-owned per-pipe data. 485 * The driver never accesses this memory after zeroing it. 486 * It doesn't even have to be saved/restored on suspend/resume 487 * because it contains no data when there are no active queues. 488 */ 489 490 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num, 491 &dqm->pipeline_mem); 492 493 if (err) { 494 pr_err("kfd: error allocate vidmem num pipes: %d\n", 495 pipes_num); 496 return -ENOMEM; 497 } 498 499 hpdptr = dqm->pipeline_mem->cpu_ptr; 500 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr; 501 502 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num); 503 504 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE); 505 if (mqd == NULL) { 506 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 507 return -ENOMEM; 508 } 509 510 for (i = 0; i < pipes_num; i++) { 511 inx = i + first_pipe; 512 /* 513 * HPD buffer on GTT is allocated by amdkfd, no need to waste 514 * space in GTT for pipelines we don't initialize 515 */ 516 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES; 517 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr); 518 /* = log2(bytes/4)-1 */ 519 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx, 520 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr); 521 } 522 523 return 0; 524} 525 526static int init_scheduler(struct device_queue_manager *dqm) 527{ 528 int retval; 529 530 BUG_ON(!dqm); 531 532 pr_debug("kfd: In %s\n", __func__); 533 534 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm)); 535 return retval; 536} 537 538static int initialize_nocpsch(struct device_queue_manager *dqm) 539{ 540 int i; 541 542 BUG_ON(!dqm); 543 544 pr_debug("kfd: In func %s num of pipes: %d\n", 545 __func__, get_pipes_num(dqm)); 546 547 mutex_init(&dqm->lock); 548 INIT_LIST_HEAD(&dqm->queues); 549 dqm->queue_count = dqm->next_pipe_to_allocate = 0; 550 dqm->sdma_queue_count = 0; 551 dqm->allocated_queues = kcalloc(get_pipes_num(dqm), 552 sizeof(unsigned int), GFP_KERNEL); 553 if (!dqm->allocated_queues) { 554 mutex_destroy(&dqm->lock); 555 return -ENOMEM; 556 } 557 558 for (i = 0; i < get_pipes_num(dqm); i++) 559 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1; 560 561 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1; 562 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1; 563 564 init_scheduler(dqm); 565 return 0; 566} 567 568static void uninitialize_nocpsch(struct device_queue_manager *dqm) 569{ 570 int i; 571 572 BUG_ON(!dqm); 573 574 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0); 575 576 kfree(dqm->allocated_queues); 577 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) 578 kfree(dqm->mqds[i]); 579 mutex_destroy(&dqm->lock); 580 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); 581} 582 583static int start_nocpsch(struct device_queue_manager *dqm) 584{ 585 return 0; 586} 587 588static int stop_nocpsch(struct device_queue_manager *dqm) 589{ 590 return 0; 591} 592 593static int allocate_sdma_queue(struct device_queue_manager *dqm, 594 unsigned int *sdma_queue_id) 595{ 596 int bit; 597 598 if (dqm->sdma_bitmap == 0) 599 return -ENOMEM; 600 601 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap, 602 CIK_SDMA_QUEUES); 603 604 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap); 605 *sdma_queue_id = bit; 606 607 return 0; 608} 609 610static void deallocate_sdma_queue(struct device_queue_manager *dqm, 611 unsigned int sdma_queue_id) 612{ 613 if (sdma_queue_id >= CIK_SDMA_QUEUES) 614 return; 615 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap); 616} 617 618static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, 619 struct qcm_process_device *qpd) 620{ 621 uint32_t value = SDMA_ATC; 622 623 if (q->process->is_32bit_user_mode) 624 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd)); 625 else 626 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64( 627 qpd_to_pdd(qpd))); 628 q->properties.sdma_vm_addr = value; 629} 630 631static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, 632 struct queue *q, 633 struct qcm_process_device *qpd) 634{ 635 struct mqd_manager *mqd; 636 int retval; 637 638 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA); 639 if (!mqd) 640 return -ENOMEM; 641 642 retval = allocate_sdma_queue(dqm, &q->sdma_id); 643 if (retval != 0) 644 return retval; 645 646 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE; 647 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM; 648 649 pr_debug("kfd: sdma id is: %d\n", q->sdma_id); 650 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id); 651 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id); 652 653 init_sdma_vm(dqm, q, qpd); 654 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 655 &q->gart_mqd_addr, &q->properties); 656 if (retval != 0) { 657 deallocate_sdma_queue(dqm, q->sdma_id); 658 return retval; 659 } 660 661 retval = mqd->load_mqd(mqd, q->mqd, 0, 662 0, NULL); 663 if (retval != 0) { 664 deallocate_sdma_queue(dqm, q->sdma_id); 665 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 666 return retval; 667 } 668 669 return 0; 670} 671 672/* 673 * Device Queue Manager implementation for cp scheduler 674 */ 675 676static int set_sched_resources(struct device_queue_manager *dqm) 677{ 678 struct scheduling_resources res; 679 unsigned int queue_num, queue_mask; 680 681 BUG_ON(!dqm); 682 683 pr_debug("kfd: In func %s\n", __func__); 684 685 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE; 686 queue_mask = (1 << queue_num) - 1; 687 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1; 688 res.vmid_mask <<= KFD_VMID_START_OFFSET; 689 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE); 690 res.gws_mask = res.oac_mask = res.gds_heap_base = 691 res.gds_heap_size = 0; 692 693 pr_debug("kfd: scheduling resources:\n" 694 " vmid mask: 0x%8X\n" 695 " queue mask: 0x%8llX\n", 696 res.vmid_mask, res.queue_mask); 697 698 return pm_send_set_resources(&dqm->packets, &res); 699} 700 701static int initialize_cpsch(struct device_queue_manager *dqm) 702{ 703 int retval; 704 705 BUG_ON(!dqm); 706 707 pr_debug("kfd: In func %s num of pipes: %d\n", 708 __func__, get_pipes_num_cpsch()); 709 710 mutex_init(&dqm->lock); 711 INIT_LIST_HEAD(&dqm->queues); 712 dqm->queue_count = dqm->processes_count = 0; 713 dqm->sdma_queue_count = 0; 714 dqm->active_runlist = false; 715 retval = dqm->ops_asic_specific.initialize(dqm); 716 if (retval != 0) 717 goto fail_init_pipelines; 718 719 return 0; 720 721fail_init_pipelines: 722 mutex_destroy(&dqm->lock); 723 return retval; 724} 725 726static int start_cpsch(struct device_queue_manager *dqm) 727{ 728 struct device_process_node *node; 729 int retval; 730 731 BUG_ON(!dqm); 732 733 retval = 0; 734 735 retval = pm_init(&dqm->packets, dqm); 736 if (retval != 0) 737 goto fail_packet_manager_init; 738 739 retval = set_sched_resources(dqm); 740 if (retval != 0) 741 goto fail_set_sched_resources; 742 743 pr_debug("kfd: allocating fence memory\n"); 744 745 /* allocate fence memory on the gart */ 746 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr), 747 &dqm->fence_mem); 748 749 if (retval != 0) 750 goto fail_allocate_vidmem; 751 752 dqm->fence_addr = dqm->fence_mem->cpu_ptr; 753 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr; 754 list_for_each_entry(node, &dqm->queues, list) 755 if (node->qpd->pqm->process && dqm->dev) 756 kfd_bind_process_to_device(dqm->dev, 757 node->qpd->pqm->process); 758 759 execute_queues_cpsch(dqm, true); 760 761 return 0; 762fail_allocate_vidmem: 763fail_set_sched_resources: 764 pm_uninit(&dqm->packets); 765fail_packet_manager_init: 766 return retval; 767} 768 769static int stop_cpsch(struct device_queue_manager *dqm) 770{ 771 struct device_process_node *node; 772 struct kfd_process_device *pdd; 773 774 BUG_ON(!dqm); 775 776 destroy_queues_cpsch(dqm, true); 777 778 list_for_each_entry(node, &dqm->queues, list) { 779 pdd = qpd_to_pdd(node->qpd); 780 pdd->bound = false; 781 } 782 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); 783 pm_uninit(&dqm->packets); 784 785 return 0; 786} 787 788static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, 789 struct kernel_queue *kq, 790 struct qcm_process_device *qpd) 791{ 792 BUG_ON(!dqm || !kq || !qpd); 793 794 pr_debug("kfd: In func %s\n", __func__); 795 796 mutex_lock(&dqm->lock); 797 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 798 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n", 799 dqm->total_queue_count); 800 mutex_unlock(&dqm->lock); 801 return -EPERM; 802 } 803 804 /* 805 * Unconditionally increment this counter, regardless of the queue's 806 * type or whether the queue is active. 807 */ 808 dqm->total_queue_count++; 809 pr_debug("Total of %d queues are accountable so far\n", 810 dqm->total_queue_count); 811 812 list_add(&kq->list, &qpd->priv_queue_list); 813 dqm->queue_count++; 814 qpd->is_debug = true; 815 execute_queues_cpsch(dqm, false); 816 mutex_unlock(&dqm->lock); 817 818 return 0; 819} 820 821static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, 822 struct kernel_queue *kq, 823 struct qcm_process_device *qpd) 824{ 825 BUG_ON(!dqm || !kq); 826 827 pr_debug("kfd: In %s\n", __func__); 828 829 mutex_lock(&dqm->lock); 830 destroy_queues_cpsch(dqm, false); 831 list_del(&kq->list); 832 dqm->queue_count--; 833 qpd->is_debug = false; 834 execute_queues_cpsch(dqm, false); 835 /* 836 * Unconditionally decrement this counter, regardless of the queue's 837 * type. 838 */ 839 dqm->total_queue_count--; 840 pr_debug("Total of %d queues are accountable so far\n", 841 dqm->total_queue_count); 842 mutex_unlock(&dqm->lock); 843} 844 845static void select_sdma_engine_id(struct queue *q) 846{ 847 static int sdma_id; 848 849 q->sdma_id = sdma_id; 850 sdma_id = (sdma_id + 1) % 2; 851} 852 853static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, 854 struct qcm_process_device *qpd, int *allocate_vmid) 855{ 856 int retval; 857 struct mqd_manager *mqd; 858 859 BUG_ON(!dqm || !q || !qpd); 860 861 retval = 0; 862 863 if (allocate_vmid) 864 *allocate_vmid = 0; 865 866 mutex_lock(&dqm->lock); 867 868 if (dqm->total_queue_count >= max_num_of_queues_per_device) { 869 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n", 870 dqm->total_queue_count); 871 retval = -EPERM; 872 goto out; 873 } 874 875 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 876 select_sdma_engine_id(q); 877 878 mqd = dqm->ops.get_mqd_manager(dqm, 879 get_mqd_type_from_queue_type(q->properties.type)); 880 881 if (mqd == NULL) { 882 mutex_unlock(&dqm->lock); 883 return -ENOMEM; 884 } 885 886 init_sdma_vm(dqm, q, qpd); 887 888 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj, 889 &q->gart_mqd_addr, &q->properties); 890 if (retval != 0) 891 goto out; 892 893 list_add(&q->list, &qpd->queues_list); 894 if (q->properties.is_active) { 895 dqm->queue_count++; 896 retval = execute_queues_cpsch(dqm, false); 897 } 898 899 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 900 dqm->sdma_queue_count++; 901 /* 902 * Unconditionally increment this counter, regardless of the queue's 903 * type or whether the queue is active. 904 */ 905 dqm->total_queue_count++; 906 907 pr_debug("Total of %d queues are accountable so far\n", 908 dqm->total_queue_count); 909 910out: 911 mutex_unlock(&dqm->lock); 912 return retval; 913} 914 915static int amdkfd_fence_wait_timeout(unsigned int *fence_addr, 916 unsigned int fence_value, 917 unsigned long timeout) 918{ 919 BUG_ON(!fence_addr); 920 timeout += jiffies; 921 922 while (*fence_addr != fence_value) { 923 if (time_after(jiffies, timeout)) { 924 pr_err("kfd: qcm fence wait loop timeout expired\n"); 925 return -ETIME; 926 } 927 schedule(); 928 } 929 930 return 0; 931} 932 933static int destroy_sdma_queues(struct device_queue_manager *dqm, 934 unsigned int sdma_engine) 935{ 936 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, 937 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 938 sdma_engine); 939} 940 941static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock) 942{ 943 int retval; 944 945 BUG_ON(!dqm); 946 947 retval = 0; 948 949 if (lock) 950 mutex_lock(&dqm->lock); 951 if (dqm->active_runlist == false) 952 goto out; 953 954 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n", 955 dqm->sdma_queue_count); 956 957 if (dqm->sdma_queue_count > 0) { 958 destroy_sdma_queues(dqm, 0); 959 destroy_sdma_queues(dqm, 1); 960 } 961 962 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, 963 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0); 964 if (retval != 0) 965 goto out; 966 967 *dqm->fence_addr = KFD_FENCE_INIT; 968 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr, 969 KFD_FENCE_COMPLETED); 970 /* should be timed out */ 971 amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED, 972 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS); 973 pm_release_ib(&dqm->packets); 974 dqm->active_runlist = false; 975 976out: 977 if (lock) 978 mutex_unlock(&dqm->lock); 979 return retval; 980} 981 982static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock) 983{ 984 int retval; 985 986 BUG_ON(!dqm); 987 988 if (lock) 989 mutex_lock(&dqm->lock); 990 991 retval = destroy_queues_cpsch(dqm, false); 992 if (retval != 0) { 993 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption"); 994 goto out; 995 } 996 997 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) { 998 retval = 0; 999 goto out; 1000 } 1001 1002 if (dqm->active_runlist) { 1003 retval = 0; 1004 goto out; 1005 } 1006 1007 retval = pm_send_runlist(&dqm->packets, &dqm->queues); 1008 if (retval != 0) { 1009 pr_err("kfd: failed to execute runlist"); 1010 goto out; 1011 } 1012 dqm->active_runlist = true; 1013 1014out: 1015 if (lock) 1016 mutex_unlock(&dqm->lock); 1017 return retval; 1018} 1019 1020static int destroy_queue_cpsch(struct device_queue_manager *dqm, 1021 struct qcm_process_device *qpd, 1022 struct queue *q) 1023{ 1024 int retval; 1025 struct mqd_manager *mqd; 1026 1027 BUG_ON(!dqm || !qpd || !q); 1028 1029 retval = 0; 1030 1031 /* remove queue from list to prevent rescheduling after preemption */ 1032 mutex_lock(&dqm->lock); 1033 mqd = dqm->ops.get_mqd_manager(dqm, 1034 get_mqd_type_from_queue_type(q->properties.type)); 1035 if (!mqd) { 1036 retval = -ENOMEM; 1037 goto failed; 1038 } 1039 1040 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) 1041 dqm->sdma_queue_count--; 1042 1043 list_del(&q->list); 1044 if (q->properties.is_active) 1045 dqm->queue_count--; 1046 1047 execute_queues_cpsch(dqm, false); 1048 1049 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 1050 1051 /* 1052 * Unconditionally decrement this counter, regardless of the queue's 1053 * type 1054 */ 1055 dqm->total_queue_count--; 1056 pr_debug("Total of %d queues are accountable so far\n", 1057 dqm->total_queue_count); 1058 1059 mutex_unlock(&dqm->lock); 1060 1061 return 0; 1062 1063failed: 1064 mutex_unlock(&dqm->lock); 1065 return retval; 1066} 1067 1068/* 1069 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to 1070 * stay in user mode. 1071 */ 1072#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL 1073/* APE1 limit is inclusive and 64K aligned. */ 1074#define APE1_LIMIT_ALIGNMENT 0xFFFF 1075 1076static bool set_cache_memory_policy(struct device_queue_manager *dqm, 1077 struct qcm_process_device *qpd, 1078 enum cache_policy default_policy, 1079 enum cache_policy alternate_policy, 1080 void __user *alternate_aperture_base, 1081 uint64_t alternate_aperture_size) 1082{ 1083 bool retval; 1084 1085 pr_debug("kfd: In func %s\n", __func__); 1086 1087 mutex_lock(&dqm->lock); 1088 1089 if (alternate_aperture_size == 0) { 1090 /* base > limit disables APE1 */ 1091 qpd->sh_mem_ape1_base = 1; 1092 qpd->sh_mem_ape1_limit = 0; 1093 } else { 1094 /* 1095 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]}, 1096 * SH_MEM_APE1_BASE[31:0], 0x0000 } 1097 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]}, 1098 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF } 1099 * Verify that the base and size parameters can be 1100 * represented in this format and convert them. 1101 * Additionally restrict APE1 to user-mode addresses. 1102 */ 1103 1104 uint64_t base = (uintptr_t)alternate_aperture_base; 1105 uint64_t limit = base + alternate_aperture_size - 1; 1106 1107 if (limit <= base) 1108 goto out; 1109 1110 if ((base & APE1_FIXED_BITS_MASK) != 0) 1111 goto out; 1112 1113 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) 1114 goto out; 1115 1116 qpd->sh_mem_ape1_base = base >> 16; 1117 qpd->sh_mem_ape1_limit = limit >> 16; 1118 } 1119 1120 retval = dqm->ops_asic_specific.set_cache_memory_policy( 1121 dqm, 1122 qpd, 1123 default_policy, 1124 alternate_policy, 1125 alternate_aperture_base, 1126 alternate_aperture_size); 1127 1128 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0)) 1129 program_sh_mem_settings(dqm, qpd); 1130 1131 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n", 1132 qpd->sh_mem_config, qpd->sh_mem_ape1_base, 1133 qpd->sh_mem_ape1_limit); 1134 1135 mutex_unlock(&dqm->lock); 1136 return retval; 1137 1138out: 1139 mutex_unlock(&dqm->lock); 1140 return false; 1141} 1142 1143struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev) 1144{ 1145 struct device_queue_manager *dqm; 1146 1147 BUG_ON(!dev); 1148 1149 pr_debug("kfd: loading device queue manager\n"); 1150 1151 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL); 1152 if (!dqm) 1153 return NULL; 1154 1155 dqm->dev = dev; 1156 switch (sched_policy) { 1157 case KFD_SCHED_POLICY_HWS: 1158 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: 1159 /* initialize dqm for cp scheduling */ 1160 dqm->ops.create_queue = create_queue_cpsch; 1161 dqm->ops.initialize = initialize_cpsch; 1162 dqm->ops.start = start_cpsch; 1163 dqm->ops.stop = stop_cpsch; 1164 dqm->ops.destroy_queue = destroy_queue_cpsch; 1165 dqm->ops.update_queue = update_queue; 1166 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1167 dqm->ops.register_process = register_process_nocpsch; 1168 dqm->ops.unregister_process = unregister_process_nocpsch; 1169 dqm->ops.uninitialize = uninitialize_nocpsch; 1170 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch; 1171 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch; 1172 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1173 break; 1174 case KFD_SCHED_POLICY_NO_HWS: 1175 /* initialize dqm for no cp scheduling */ 1176 dqm->ops.start = start_nocpsch; 1177 dqm->ops.stop = stop_nocpsch; 1178 dqm->ops.create_queue = create_queue_nocpsch; 1179 dqm->ops.destroy_queue = destroy_queue_nocpsch; 1180 dqm->ops.update_queue = update_queue; 1181 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch; 1182 dqm->ops.register_process = register_process_nocpsch; 1183 dqm->ops.unregister_process = unregister_process_nocpsch; 1184 dqm->ops.initialize = initialize_nocpsch; 1185 dqm->ops.uninitialize = uninitialize_nocpsch; 1186 dqm->ops.set_cache_memory_policy = set_cache_memory_policy; 1187 break; 1188 default: 1189 BUG(); 1190 break; 1191 } 1192 1193 switch (dev->device_info->asic_family) { 1194 case CHIP_CARRIZO: 1195 device_queue_manager_init_vi(&dqm->ops_asic_specific); 1196 break; 1197 1198 case CHIP_KAVERI: 1199 device_queue_manager_init_cik(&dqm->ops_asic_specific); 1200 break; 1201 } 1202 1203 if (dqm->ops.initialize(dqm) != 0) { 1204 kfree(dqm); 1205 return NULL; 1206 } 1207 1208 return dqm; 1209} 1210 1211void device_queue_manager_uninit(struct device_queue_manager *dqm) 1212{ 1213 BUG_ON(!dqm); 1214 1215 dqm->ops.uninitialize(dqm); 1216 kfree(dqm); 1217} 1218