1/* 2 * Keystone accumulator queue manager 3 * 4 * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com 5 * Author: Sandeep Nair <sandeep_n@ti.com> 6 * Cyril Chemparathy <cyril@ti.com> 7 * Santosh Shilimkar <santosh.shilimkar@ti.com> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * version 2 as published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 */ 18 19#include <linux/kernel.h> 20#include <linux/module.h> 21#include <linux/device.h> 22#include <linux/io.h> 23#include <linux/interrupt.h> 24#include <linux/bitops.h> 25#include <linux/slab.h> 26#include <linux/spinlock.h> 27#include <linux/soc/ti/knav_qmss.h> 28#include <linux/platform_device.h> 29#include <linux/dma-mapping.h> 30#include <linux/of.h> 31#include <linux/of_device.h> 32#include <linux/of_address.h> 33#include <linux/firmware.h> 34 35#include "knav_qmss.h" 36 37#define knav_range_offset_to_inst(kdev, range, q) \ 38 (range->queue_base_inst + (q << kdev->inst_shift)) 39 40static void __knav_acc_notify(struct knav_range_info *range, 41 struct knav_acc_channel *acc) 42{ 43 struct knav_device *kdev = range->kdev; 44 struct knav_queue_inst *inst; 45 int range_base, queue; 46 47 range_base = kdev->base_id + range->queue_base; 48 49 if (range->flags & RANGE_MULTI_QUEUE) { 50 for (queue = 0; queue < range->num_queues; queue++) { 51 inst = knav_range_offset_to_inst(kdev, range, 52 queue); 53 if (inst->notify_needed) { 54 inst->notify_needed = 0; 55 dev_dbg(kdev->dev, "acc-irq: notifying %d\n", 56 range_base + queue); 57 knav_queue_notify(inst); 58 } 59 } 60 } else { 61 queue = acc->channel - range->acc_info.start_channel; 62 inst = knav_range_offset_to_inst(kdev, range, queue); 63 dev_dbg(kdev->dev, "acc-irq: notifying %d\n", 64 range_base + queue); 65 knav_queue_notify(inst); 66 } 67} 68 69static int knav_acc_set_notify(struct knav_range_info *range, 70 struct knav_queue_inst *kq, 71 bool enabled) 72{ 73 struct knav_pdsp_info *pdsp = range->acc_info.pdsp; 74 struct knav_device *kdev = range->kdev; 75 u32 mask, offset; 76 77 /* 78 * when enabling, we need to re-trigger an interrupt if we 79 * have descriptors pending 80 */ 81 if (!enabled || atomic_read(&kq->desc_count) <= 0) 82 return 0; 83 84 kq->notify_needed = 1; 85 atomic_inc(&kq->acc->retrigger_count); 86 mask = BIT(kq->acc->channel % 32); 87 offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel); 88 dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n", 89 kq->acc->name); 90 writel_relaxed(mask, pdsp->intd + offset); 91 return 0; 92} 93 94static irqreturn_t knav_acc_int_handler(int irq, void *_instdata) 95{ 96 struct knav_acc_channel *acc; 97 struct knav_queue_inst *kq = NULL; 98 struct knav_range_info *range; 99 struct knav_pdsp_info *pdsp; 100 struct knav_acc_info *info; 101 struct knav_device *kdev; 102 103 u32 *list, *list_cpu, val, idx, notifies; 104 int range_base, channel, queue = 0; 105 dma_addr_t list_dma; 106 107 range = _instdata; 108 info = &range->acc_info; 109 kdev = range->kdev; 110 pdsp = range->acc_info.pdsp; 111 acc = range->acc; 112 113 range_base = kdev->base_id + range->queue_base; 114 if ((range->flags & RANGE_MULTI_QUEUE) == 0) { 115 for (queue = 0; queue < range->num_irqs; queue++) 116 if (range->irqs[queue].irq == irq) 117 break; 118 kq = knav_range_offset_to_inst(kdev, range, queue); 119 acc += queue; 120 } 121 122 channel = acc->channel; 123 list_dma = acc->list_dma[acc->list_index]; 124 list_cpu = acc->list_cpu[acc->list_index]; 125 dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, phys %x\n", 126 channel, acc->list_index, list_cpu, list_dma); 127 if (atomic_read(&acc->retrigger_count)) { 128 atomic_dec(&acc->retrigger_count); 129 __knav_acc_notify(range, acc); 130 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel)); 131 /* ack the interrupt */ 132 writel_relaxed(ACC_CHANNEL_INT_BASE + channel, 133 pdsp->intd + ACC_INTD_OFFSET_EOI); 134 135 return IRQ_HANDLED; 136 } 137 138 notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel)); 139 WARN_ON(!notifies); 140 dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size, 141 DMA_FROM_DEVICE); 142 143 for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32)); 144 list += ACC_LIST_ENTRY_WORDS) { 145 if (ACC_LIST_ENTRY_WORDS == 1) { 146 dev_dbg(kdev->dev, 147 "acc-irq: list %d, entry @%p, %08x\n", 148 acc->list_index, list, list[0]); 149 } else if (ACC_LIST_ENTRY_WORDS == 2) { 150 dev_dbg(kdev->dev, 151 "acc-irq: list %d, entry @%p, %08x %08x\n", 152 acc->list_index, list, list[0], list[1]); 153 } else if (ACC_LIST_ENTRY_WORDS == 4) { 154 dev_dbg(kdev->dev, 155 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n", 156 acc->list_index, list, list[0], list[1], 157 list[2], list[3]); 158 } 159 160 val = list[ACC_LIST_ENTRY_DESC_IDX]; 161 if (!val) 162 break; 163 164 if (range->flags & RANGE_MULTI_QUEUE) { 165 queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16; 166 if (queue < range_base || 167 queue >= range_base + range->num_queues) { 168 dev_err(kdev->dev, 169 "bad queue %d, expecting %d-%d\n", 170 queue, range_base, 171 range_base + range->num_queues); 172 break; 173 } 174 queue -= range_base; 175 kq = knav_range_offset_to_inst(kdev, range, 176 queue); 177 } 178 179 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) { 180 atomic_dec(&kq->desc_count); 181 dev_err(kdev->dev, 182 "acc-irq: queue %d full, entry dropped\n", 183 queue + range_base); 184 continue; 185 } 186 187 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK; 188 kq->descs[idx] = val; 189 kq->notify_needed = 1; 190 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n", 191 val, idx, queue + range_base); 192 } 193 194 __knav_acc_notify(range, acc); 195 memset(list_cpu, 0, info->list_size); 196 dma_sync_single_for_device(kdev->dev, list_dma, info->list_size, 197 DMA_TO_DEVICE); 198 199 /* flip to the other list */ 200 acc->list_index ^= 1; 201 202 /* reset the interrupt counter */ 203 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel)); 204 205 /* ack the interrupt */ 206 writel_relaxed(ACC_CHANNEL_INT_BASE + channel, 207 pdsp->intd + ACC_INTD_OFFSET_EOI); 208 209 return IRQ_HANDLED; 210} 211 212static int knav_range_setup_acc_irq(struct knav_range_info *range, 213 int queue, bool enabled) 214{ 215 struct knav_device *kdev = range->kdev; 216 struct knav_acc_channel *acc; 217 unsigned long cpu_map; 218 int ret = 0, irq; 219 u32 old, new; 220 221 if (range->flags & RANGE_MULTI_QUEUE) { 222 acc = range->acc; 223 irq = range->irqs[0].irq; 224 cpu_map = range->irqs[0].cpu_map; 225 } else { 226 acc = range->acc + queue; 227 irq = range->irqs[queue].irq; 228 cpu_map = range->irqs[queue].cpu_map; 229 } 230 231 old = acc->open_mask; 232 if (enabled) 233 new = old | BIT(queue); 234 else 235 new = old & ~BIT(queue); 236 acc->open_mask = new; 237 238 dev_dbg(kdev->dev, 239 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n", 240 old, new, acc->name); 241 242 if (likely(new == old)) 243 return 0; 244 245 if (new && !old) { 246 dev_dbg(kdev->dev, 247 "setup-acc-irq: requesting %s for channel %s\n", 248 acc->name, acc->name); 249 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name, 250 range); 251 if (!ret && cpu_map) { 252 ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map)); 253 if (ret) { 254 dev_warn(range->kdev->dev, 255 "Failed to set IRQ affinity\n"); 256 return ret; 257 } 258 } 259 } 260 261 if (old && !new) { 262 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n", 263 acc->name, acc->name); 264 ret = irq_set_affinity_hint(irq, NULL); 265 if (ret) 266 dev_warn(range->kdev->dev, 267 "Failed to set IRQ affinity\n"); 268 free_irq(irq, range); 269 } 270 271 return ret; 272} 273 274static const char *knav_acc_result_str(enum knav_acc_result result) 275{ 276 static const char * const result_str[] = { 277 [ACC_RET_IDLE] = "idle", 278 [ACC_RET_SUCCESS] = "success", 279 [ACC_RET_INVALID_COMMAND] = "invalid command", 280 [ACC_RET_INVALID_CHANNEL] = "invalid channel", 281 [ACC_RET_INACTIVE_CHANNEL] = "inactive channel", 282 [ACC_RET_ACTIVE_CHANNEL] = "active channel", 283 [ACC_RET_INVALID_QUEUE] = "invalid queue", 284 [ACC_RET_INVALID_RET] = "invalid return code", 285 }; 286 287 if (result >= ARRAY_SIZE(result_str)) 288 return result_str[ACC_RET_INVALID_RET]; 289 else 290 return result_str[result]; 291} 292 293static enum knav_acc_result 294knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp, 295 struct knav_reg_acc_command *cmd) 296{ 297 u32 result; 298 299 dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n", 300 cmd->command, cmd->queue_mask, cmd->list_phys, 301 cmd->queue_num, cmd->timer_config); 302 303 writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config); 304 writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num); 305 writel_relaxed(cmd->list_phys, &pdsp->acc_command->list_phys); 306 writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask); 307 writel_relaxed(cmd->command, &pdsp->acc_command->command); 308 309 /* wait for the command to clear */ 310 do { 311 result = readl_relaxed(&pdsp->acc_command->command); 312 } while ((result >> 8) & 0xff); 313 314 return (result >> 24) & 0xff; 315} 316 317static void knav_acc_setup_cmd(struct knav_device *kdev, 318 struct knav_range_info *range, 319 struct knav_reg_acc_command *cmd, 320 int queue) 321{ 322 struct knav_acc_info *info = &range->acc_info; 323 struct knav_acc_channel *acc; 324 int queue_base; 325 u32 queue_mask; 326 327 if (range->flags & RANGE_MULTI_QUEUE) { 328 acc = range->acc; 329 queue_base = range->queue_base; 330 queue_mask = BIT(range->num_queues) - 1; 331 } else { 332 acc = range->acc + queue; 333 queue_base = range->queue_base + queue; 334 queue_mask = 0; 335 } 336 337 memset(cmd, 0, sizeof(*cmd)); 338 cmd->command = acc->channel; 339 cmd->queue_mask = queue_mask; 340 cmd->list_phys = acc->list_dma[0]; 341 cmd->queue_num = info->list_entries << 16; 342 cmd->queue_num |= queue_base; 343 344 cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18; 345 if (range->flags & RANGE_MULTI_QUEUE) 346 cmd->timer_config |= ACC_CFG_MULTI_QUEUE; 347 cmd->timer_config |= info->pacing_mode << 16; 348 cmd->timer_config |= info->timer_count; 349} 350 351static void knav_acc_stop(struct knav_device *kdev, 352 struct knav_range_info *range, 353 int queue) 354{ 355 struct knav_reg_acc_command cmd; 356 struct knav_acc_channel *acc; 357 enum knav_acc_result result; 358 359 acc = range->acc + queue; 360 361 knav_acc_setup_cmd(kdev, range, &cmd, queue); 362 cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8; 363 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd); 364 365 dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n", 366 acc->name, knav_acc_result_str(result)); 367} 368 369static enum knav_acc_result knav_acc_start(struct knav_device *kdev, 370 struct knav_range_info *range, 371 int queue) 372{ 373 struct knav_reg_acc_command cmd; 374 struct knav_acc_channel *acc; 375 enum knav_acc_result result; 376 377 acc = range->acc + queue; 378 379 knav_acc_setup_cmd(kdev, range, &cmd, queue); 380 cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8; 381 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd); 382 383 dev_dbg(kdev->dev, "started acc channel %s, result %s\n", 384 acc->name, knav_acc_result_str(result)); 385 386 return result; 387} 388 389static int knav_acc_init_range(struct knav_range_info *range) 390{ 391 struct knav_device *kdev = range->kdev; 392 struct knav_acc_channel *acc; 393 enum knav_acc_result result; 394 int queue; 395 396 for (queue = 0; queue < range->num_queues; queue++) { 397 acc = range->acc + queue; 398 399 knav_acc_stop(kdev, range, queue); 400 acc->list_index = 0; 401 result = knav_acc_start(kdev, range, queue); 402 403 if (result != ACC_RET_SUCCESS) 404 return -EIO; 405 406 if (range->flags & RANGE_MULTI_QUEUE) 407 return 0; 408 } 409 return 0; 410} 411 412static int knav_acc_init_queue(struct knav_range_info *range, 413 struct knav_queue_inst *kq) 414{ 415 unsigned id = kq->id - range->queue_base; 416 417 kq->descs = devm_kzalloc(range->kdev->dev, 418 ACC_DESCS_MAX * sizeof(u32), GFP_KERNEL); 419 if (!kq->descs) 420 return -ENOMEM; 421 422 kq->acc = range->acc; 423 if ((range->flags & RANGE_MULTI_QUEUE) == 0) 424 kq->acc += id; 425 return 0; 426} 427 428static int knav_acc_open_queue(struct knav_range_info *range, 429 struct knav_queue_inst *inst, unsigned flags) 430{ 431 unsigned id = inst->id - range->queue_base; 432 433 return knav_range_setup_acc_irq(range, id, true); 434} 435 436static int knav_acc_close_queue(struct knav_range_info *range, 437 struct knav_queue_inst *inst) 438{ 439 unsigned id = inst->id - range->queue_base; 440 441 return knav_range_setup_acc_irq(range, id, false); 442} 443 444static int knav_acc_free_range(struct knav_range_info *range) 445{ 446 struct knav_device *kdev = range->kdev; 447 struct knav_acc_channel *acc; 448 struct knav_acc_info *info; 449 int channel, channels; 450 451 info = &range->acc_info; 452 453 if (range->flags & RANGE_MULTI_QUEUE) 454 channels = 1; 455 else 456 channels = range->num_queues; 457 458 for (channel = 0; channel < channels; channel++) { 459 acc = range->acc + channel; 460 if (!acc->list_cpu[0]) 461 continue; 462 dma_unmap_single(kdev->dev, acc->list_dma[0], 463 info->mem_size, DMA_BIDIRECTIONAL); 464 free_pages_exact(acc->list_cpu[0], info->mem_size); 465 } 466 devm_kfree(range->kdev->dev, range->acc); 467 return 0; 468} 469 470struct knav_range_ops knav_acc_range_ops = { 471 .set_notify = knav_acc_set_notify, 472 .init_queue = knav_acc_init_queue, 473 .open_queue = knav_acc_open_queue, 474 .close_queue = knav_acc_close_queue, 475 .init_range = knav_acc_init_range, 476 .free_range = knav_acc_free_range, 477}; 478 479/** 480 * knav_init_acc_range: Initialise accumulator ranges 481 * 482 * @kdev: qmss device 483 * @node: device node 484 * @range: qmms range information 485 * 486 * Return 0 on success or error 487 */ 488int knav_init_acc_range(struct knav_device *kdev, 489 struct device_node *node, 490 struct knav_range_info *range) 491{ 492 struct knav_acc_channel *acc; 493 struct knav_pdsp_info *pdsp; 494 struct knav_acc_info *info; 495 int ret, channel, channels; 496 int list_size, mem_size; 497 dma_addr_t list_dma; 498 void *list_mem; 499 u32 config[5]; 500 501 range->flags |= RANGE_HAS_ACCUMULATOR; 502 info = &range->acc_info; 503 504 ret = of_property_read_u32_array(node, "accumulator", config, 5); 505 if (ret) 506 return ret; 507 508 info->pdsp_id = config[0]; 509 info->start_channel = config[1]; 510 info->list_entries = config[2]; 511 info->pacing_mode = config[3]; 512 info->timer_count = config[4] / ACC_DEFAULT_PERIOD; 513 514 if (info->start_channel > ACC_MAX_CHANNEL) { 515 dev_err(kdev->dev, "channel %d invalid for range %s\n", 516 info->start_channel, range->name); 517 return -EINVAL; 518 } 519 520 if (info->pacing_mode > 3) { 521 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n", 522 info->pacing_mode, range->name); 523 return -EINVAL; 524 } 525 526 pdsp = knav_find_pdsp(kdev, info->pdsp_id); 527 if (!pdsp) { 528 dev_err(kdev->dev, "pdsp id %d not found for range %s\n", 529 info->pdsp_id, range->name); 530 return -EINVAL; 531 } 532 533 if (!pdsp->started) { 534 dev_err(kdev->dev, "pdsp id %d not started for range %s\n", 535 info->pdsp_id, range->name); 536 return -ENODEV; 537 } 538 539 info->pdsp = pdsp; 540 channels = range->num_queues; 541 if (of_get_property(node, "multi-queue", NULL)) { 542 range->flags |= RANGE_MULTI_QUEUE; 543 channels = 1; 544 if (range->queue_base & (32 - 1)) { 545 dev_err(kdev->dev, 546 "misaligned multi-queue accumulator range %s\n", 547 range->name); 548 return -EINVAL; 549 } 550 if (range->num_queues > 32) { 551 dev_err(kdev->dev, 552 "too many queues in accumulator range %s\n", 553 range->name); 554 return -EINVAL; 555 } 556 } 557 558 /* figure out list size */ 559 list_size = info->list_entries; 560 list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32); 561 info->list_size = list_size; 562 mem_size = PAGE_ALIGN(list_size * 2); 563 info->mem_size = mem_size; 564 range->acc = devm_kzalloc(kdev->dev, channels * sizeof(*range->acc), 565 GFP_KERNEL); 566 if (!range->acc) 567 return -ENOMEM; 568 569 for (channel = 0; channel < channels; channel++) { 570 acc = range->acc + channel; 571 acc->channel = info->start_channel + channel; 572 573 /* allocate memory for the two lists */ 574 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA); 575 if (!list_mem) 576 return -ENOMEM; 577 578 list_dma = dma_map_single(kdev->dev, list_mem, mem_size, 579 DMA_BIDIRECTIONAL); 580 if (dma_mapping_error(kdev->dev, list_dma)) { 581 free_pages_exact(list_mem, mem_size); 582 return -ENOMEM; 583 } 584 585 memset(list_mem, 0, mem_size); 586 dma_sync_single_for_device(kdev->dev, list_dma, mem_size, 587 DMA_TO_DEVICE); 588 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d", 589 acc->channel); 590 acc->list_cpu[0] = list_mem; 591 acc->list_cpu[1] = list_mem + list_size; 592 acc->list_dma[0] = list_dma; 593 acc->list_dma[1] = list_dma + list_size; 594 dev_dbg(kdev->dev, "%s: channel %d, phys %08x, virt %8p\n", 595 acc->name, acc->channel, list_dma, list_mem); 596 } 597 598 range->ops = &knav_acc_range_ops; 599 return 0; 600} 601EXPORT_SYMBOL_GPL(knav_init_acc_range); 602