1/* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License version 2 as 4 * published by the Free Software Foundation. 5 */ 6 7#include <asm/cacheflush.h> 8#include <asm/pgtable.h> 9#include <linux/compiler.h> 10#include <linux/delay.h> 11#include <linux/device.h> 12#include <linux/errno.h> 13#include <linux/interrupt.h> 14#include <linux/io.h> 15#include <linux/iommu.h> 16#include <linux/jiffies.h> 17#include <linux/list.h> 18#include <linux/mm.h> 19#include <linux/module.h> 20#include <linux/of.h> 21#include <linux/of_platform.h> 22#include <linux/platform_device.h> 23#include <linux/slab.h> 24#include <linux/spinlock.h> 25 26/** MMU register offsets */ 27#define RK_MMU_DTE_ADDR 0x00 /* Directory table address */ 28#define RK_MMU_STATUS 0x04 29#define RK_MMU_COMMAND 0x08 30#define RK_MMU_PAGE_FAULT_ADDR 0x0C /* IOVA of last page fault */ 31#define RK_MMU_ZAP_ONE_LINE 0x10 /* Shootdown one IOTLB entry */ 32#define RK_MMU_INT_RAWSTAT 0x14 /* IRQ status ignoring mask */ 33#define RK_MMU_INT_CLEAR 0x18 /* Acknowledge and re-arm irq */ 34#define RK_MMU_INT_MASK 0x1C /* IRQ enable */ 35#define RK_MMU_INT_STATUS 0x20 /* IRQ status after masking */ 36#define RK_MMU_AUTO_GATING 0x24 37 38#define DTE_ADDR_DUMMY 0xCAFEBABE 39#define FORCE_RESET_TIMEOUT 100 /* ms */ 40 41/* RK_MMU_STATUS fields */ 42#define RK_MMU_STATUS_PAGING_ENABLED BIT(0) 43#define RK_MMU_STATUS_PAGE_FAULT_ACTIVE BIT(1) 44#define RK_MMU_STATUS_STALL_ACTIVE BIT(2) 45#define RK_MMU_STATUS_IDLE BIT(3) 46#define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4) 47#define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5) 48#define RK_MMU_STATUS_STALL_NOT_ACTIVE BIT(31) 49 50/* RK_MMU_COMMAND command values */ 51#define RK_MMU_CMD_ENABLE_PAGING 0 /* Enable memory translation */ 52#define RK_MMU_CMD_DISABLE_PAGING 1 /* Disable memory translation */ 53#define RK_MMU_CMD_ENABLE_STALL 2 /* Stall paging to allow other cmds */ 54#define RK_MMU_CMD_DISABLE_STALL 3 /* Stop stall re-enables paging */ 55#define RK_MMU_CMD_ZAP_CACHE 4 /* Shoot down entire IOTLB */ 56#define RK_MMU_CMD_PAGE_FAULT_DONE 5 /* Clear page fault */ 57#define RK_MMU_CMD_FORCE_RESET 6 /* Reset all registers */ 58 59/* RK_MMU_INT_* register fields */ 60#define RK_MMU_IRQ_PAGE_FAULT 0x01 /* page fault */ 61#define RK_MMU_IRQ_BUS_ERROR 0x02 /* bus read error */ 62#define RK_MMU_IRQ_MASK (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR) 63 64#define NUM_DT_ENTRIES 1024 65#define NUM_PT_ENTRIES 1024 66 67#define SPAGE_ORDER 12 68#define SPAGE_SIZE (1 << SPAGE_ORDER) 69 70 /* 71 * Support mapping any size that fits in one page table: 72 * 4 KiB to 4 MiB 73 */ 74#define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 75 76#define IOMMU_REG_POLL_COUNT_FAST 1000 77 78struct rk_iommu_domain { 79 struct list_head iommus; 80 u32 *dt; /* page directory table */ 81 spinlock_t iommus_lock; /* lock for iommus list */ 82 spinlock_t dt_lock; /* lock for modifying page directory table */ 83 84 struct iommu_domain domain; 85}; 86 87struct rk_iommu { 88 struct device *dev; 89 void __iomem *base; 90 int irq; 91 struct list_head node; /* entry in rk_iommu_domain.iommus */ 92 struct iommu_domain *domain; /* domain to which iommu is attached */ 93}; 94 95static inline void rk_table_flush(u32 *va, unsigned int count) 96{ 97 phys_addr_t pa_start = virt_to_phys(va); 98 phys_addr_t pa_end = virt_to_phys(va + count); 99 size_t size = pa_end - pa_start; 100 101 __cpuc_flush_dcache_area(va, size); 102 outer_flush_range(pa_start, pa_end); 103} 104 105static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 106{ 107 return container_of(dom, struct rk_iommu_domain, domain); 108} 109 110/** 111 * Inspired by _wait_for in intel_drv.h 112 * This is NOT safe for use in interrupt context. 113 * 114 * Note that it's important that we check the condition again after having 115 * timed out, since the timeout could be due to preemption or similar and 116 * we've never had a chance to check the condition before the timeout. 117 */ 118#define rk_wait_for(COND, MS) ({ \ 119 unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ 120 int ret__ = 0; \ 121 while (!(COND)) { \ 122 if (time_after(jiffies, timeout__)) { \ 123 ret__ = (COND) ? 0 : -ETIMEDOUT; \ 124 break; \ 125 } \ 126 usleep_range(50, 100); \ 127 } \ 128 ret__; \ 129}) 130 131/* 132 * The Rockchip rk3288 iommu uses a 2-level page table. 133 * The first level is the "Directory Table" (DT). 134 * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing 135 * to a "Page Table". 136 * The second level is the 1024 Page Tables (PT). 137 * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to 138 * a 4 KB page of physical memory. 139 * 140 * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries). 141 * Each iommu device has a MMU_DTE_ADDR register that contains the physical 142 * address of the start of the DT page. 143 * 144 * The structure of the page table is as follows: 145 * 146 * DT 147 * MMU_DTE_ADDR -> +-----+ 148 * | | 149 * +-----+ PT 150 * | DTE | -> +-----+ 151 * +-----+ | | Memory 152 * | | +-----+ Page 153 * | | | PTE | -> +-----+ 154 * +-----+ +-----+ | | 155 * | | | | 156 * | | | | 157 * +-----+ | | 158 * | | 159 * | | 160 * +-----+ 161 */ 162 163/* 164 * Each DTE has a PT address and a valid bit: 165 * +---------------------+-----------+-+ 166 * | PT address | Reserved |V| 167 * +---------------------+-----------+-+ 168 * 31:12 - PT address (PTs always starts on a 4 KB boundary) 169 * 11: 1 - Reserved 170 * 0 - 1 if PT @ PT address is valid 171 */ 172#define RK_DTE_PT_ADDRESS_MASK 0xfffff000 173#define RK_DTE_PT_VALID BIT(0) 174 175static inline phys_addr_t rk_dte_pt_address(u32 dte) 176{ 177 return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK; 178} 179 180static inline bool rk_dte_is_pt_valid(u32 dte) 181{ 182 return dte & RK_DTE_PT_VALID; 183} 184 185static u32 rk_mk_dte(u32 *pt) 186{ 187 phys_addr_t pt_phys = virt_to_phys(pt); 188 return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; 189} 190 191/* 192 * Each PTE has a Page address, some flags and a valid bit: 193 * +---------------------+---+-------+-+ 194 * | Page address |Rsv| Flags |V| 195 * +---------------------+---+-------+-+ 196 * 31:12 - Page address (Pages always start on a 4 KB boundary) 197 * 11: 9 - Reserved 198 * 8: 1 - Flags 199 * 8 - Read allocate - allocate cache space on read misses 200 * 7 - Read cache - enable cache & prefetch of data 201 * 6 - Write buffer - enable delaying writes on their way to memory 202 * 5 - Write allocate - allocate cache space on write misses 203 * 4 - Write cache - different writes can be merged together 204 * 3 - Override cache attributes 205 * if 1, bits 4-8 control cache attributes 206 * if 0, the system bus defaults are used 207 * 2 - Writable 208 * 1 - Readable 209 * 0 - 1 if Page @ Page address is valid 210 */ 211#define RK_PTE_PAGE_ADDRESS_MASK 0xfffff000 212#define RK_PTE_PAGE_FLAGS_MASK 0x000001fe 213#define RK_PTE_PAGE_WRITABLE BIT(2) 214#define RK_PTE_PAGE_READABLE BIT(1) 215#define RK_PTE_PAGE_VALID BIT(0) 216 217static inline phys_addr_t rk_pte_page_address(u32 pte) 218{ 219 return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; 220} 221 222static inline bool rk_pte_is_page_valid(u32 pte) 223{ 224 return pte & RK_PTE_PAGE_VALID; 225} 226 227/* TODO: set cache flags per prot IOMMU_CACHE */ 228static u32 rk_mk_pte(phys_addr_t page, int prot) 229{ 230 u32 flags = 0; 231 flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; 232 flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; 233 page &= RK_PTE_PAGE_ADDRESS_MASK; 234 return page | flags | RK_PTE_PAGE_VALID; 235} 236 237static u32 rk_mk_pte_invalid(u32 pte) 238{ 239 return pte & ~RK_PTE_PAGE_VALID; 240} 241 242/* 243 * rk3288 iova (IOMMU Virtual Address) format 244 * 31 22.21 12.11 0 245 * +-----------+-----------+-------------+ 246 * | DTE index | PTE index | Page offset | 247 * +-----------+-----------+-------------+ 248 * 31:22 - DTE index - index of DTE in DT 249 * 21:12 - PTE index - index of PTE in PT @ DTE.pt_address 250 * 11: 0 - Page offset - offset into page @ PTE.page_address 251 */ 252#define RK_IOVA_DTE_MASK 0xffc00000 253#define RK_IOVA_DTE_SHIFT 22 254#define RK_IOVA_PTE_MASK 0x003ff000 255#define RK_IOVA_PTE_SHIFT 12 256#define RK_IOVA_PAGE_MASK 0x00000fff 257#define RK_IOVA_PAGE_SHIFT 0 258 259static u32 rk_iova_dte_index(dma_addr_t iova) 260{ 261 return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT; 262} 263 264static u32 rk_iova_pte_index(dma_addr_t iova) 265{ 266 return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT; 267} 268 269static u32 rk_iova_page_offset(dma_addr_t iova) 270{ 271 return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; 272} 273 274static u32 rk_iommu_read(struct rk_iommu *iommu, u32 offset) 275{ 276 return readl(iommu->base + offset); 277} 278 279static void rk_iommu_write(struct rk_iommu *iommu, u32 offset, u32 value) 280{ 281 writel(value, iommu->base + offset); 282} 283 284static void rk_iommu_command(struct rk_iommu *iommu, u32 command) 285{ 286 writel(command, iommu->base + RK_MMU_COMMAND); 287} 288 289static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, 290 size_t size) 291{ 292 dma_addr_t iova_end = iova + size; 293 /* 294 * TODO(djkurtz): Figure out when it is more efficient to shootdown the 295 * entire iotlb rather than iterate over individual iovas. 296 */ 297 for (; iova < iova_end; iova += SPAGE_SIZE) 298 rk_iommu_write(iommu, RK_MMU_ZAP_ONE_LINE, iova); 299} 300 301static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) 302{ 303 return rk_iommu_read(iommu, RK_MMU_STATUS) & RK_MMU_STATUS_STALL_ACTIVE; 304} 305 306static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) 307{ 308 return rk_iommu_read(iommu, RK_MMU_STATUS) & 309 RK_MMU_STATUS_PAGING_ENABLED; 310} 311 312static int rk_iommu_enable_stall(struct rk_iommu *iommu) 313{ 314 int ret; 315 316 if (rk_iommu_is_stall_active(iommu)) 317 return 0; 318 319 /* Stall can only be enabled if paging is enabled */ 320 if (!rk_iommu_is_paging_enabled(iommu)) 321 return 0; 322 323 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); 324 325 ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); 326 if (ret) 327 dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", 328 rk_iommu_read(iommu, RK_MMU_STATUS)); 329 330 return ret; 331} 332 333static int rk_iommu_disable_stall(struct rk_iommu *iommu) 334{ 335 int ret; 336 337 if (!rk_iommu_is_stall_active(iommu)) 338 return 0; 339 340 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); 341 342 ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); 343 if (ret) 344 dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", 345 rk_iommu_read(iommu, RK_MMU_STATUS)); 346 347 return ret; 348} 349 350static int rk_iommu_enable_paging(struct rk_iommu *iommu) 351{ 352 int ret; 353 354 if (rk_iommu_is_paging_enabled(iommu)) 355 return 0; 356 357 rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); 358 359 ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); 360 if (ret) 361 dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", 362 rk_iommu_read(iommu, RK_MMU_STATUS)); 363 364 return ret; 365} 366 367static int rk_iommu_disable_paging(struct rk_iommu *iommu) 368{ 369 int ret; 370 371 if (!rk_iommu_is_paging_enabled(iommu)) 372 return 0; 373 374 rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); 375 376 ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); 377 if (ret) 378 dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", 379 rk_iommu_read(iommu, RK_MMU_STATUS)); 380 381 return ret; 382} 383 384static int rk_iommu_force_reset(struct rk_iommu *iommu) 385{ 386 int ret; 387 u32 dte_addr; 388 389 /* 390 * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY 391 * and verifying that upper 5 nybbles are read back. 392 */ 393 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); 394 395 dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 396 if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { 397 dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); 398 return -EFAULT; 399 } 400 401 rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); 402 403 ret = rk_wait_for(rk_iommu_read(iommu, RK_MMU_DTE_ADDR) == 0x00000000, 404 FORCE_RESET_TIMEOUT); 405 if (ret) 406 dev_err(iommu->dev, "FORCE_RESET command timed out\n"); 407 408 return ret; 409} 410 411static void log_iova(struct rk_iommu *iommu, dma_addr_t iova) 412{ 413 u32 dte_index, pte_index, page_offset; 414 u32 mmu_dte_addr; 415 phys_addr_t mmu_dte_addr_phys, dte_addr_phys; 416 u32 *dte_addr; 417 u32 dte; 418 phys_addr_t pte_addr_phys = 0; 419 u32 *pte_addr = NULL; 420 u32 pte = 0; 421 phys_addr_t page_addr_phys = 0; 422 u32 page_flags = 0; 423 424 dte_index = rk_iova_dte_index(iova); 425 pte_index = rk_iova_pte_index(iova); 426 page_offset = rk_iova_page_offset(iova); 427 428 mmu_dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); 429 mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; 430 431 dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); 432 dte_addr = phys_to_virt(dte_addr_phys); 433 dte = *dte_addr; 434 435 if (!rk_dte_is_pt_valid(dte)) 436 goto print_it; 437 438 pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4); 439 pte_addr = phys_to_virt(pte_addr_phys); 440 pte = *pte_addr; 441 442 if (!rk_pte_is_page_valid(pte)) 443 goto print_it; 444 445 page_addr_phys = rk_pte_page_address(pte) + page_offset; 446 page_flags = pte & RK_PTE_PAGE_FLAGS_MASK; 447 448print_it: 449 dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n", 450 &iova, dte_index, pte_index, page_offset); 451 dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n", 452 &mmu_dte_addr_phys, &dte_addr_phys, dte, 453 rk_dte_is_pt_valid(dte), &pte_addr_phys, pte, 454 rk_pte_is_page_valid(pte), &page_addr_phys, page_flags); 455} 456 457static irqreturn_t rk_iommu_irq(int irq, void *dev_id) 458{ 459 struct rk_iommu *iommu = dev_id; 460 u32 status; 461 u32 int_status; 462 dma_addr_t iova; 463 464 int_status = rk_iommu_read(iommu, RK_MMU_INT_STATUS); 465 if (int_status == 0) 466 return IRQ_NONE; 467 468 iova = rk_iommu_read(iommu, RK_MMU_PAGE_FAULT_ADDR); 469 470 if (int_status & RK_MMU_IRQ_PAGE_FAULT) { 471 int flags; 472 473 status = rk_iommu_read(iommu, RK_MMU_STATUS); 474 flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? 475 IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; 476 477 dev_err(iommu->dev, "Page fault at %pad of type %s\n", 478 &iova, 479 (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); 480 481 log_iova(iommu, iova); 482 483 /* 484 * Report page fault to any installed handlers. 485 * Ignore the return code, though, since we always zap cache 486 * and clear the page fault anyway. 487 */ 488 if (iommu->domain) 489 report_iommu_fault(iommu->domain, iommu->dev, iova, 490 flags); 491 else 492 dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); 493 494 rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 495 rk_iommu_command(iommu, RK_MMU_CMD_PAGE_FAULT_DONE); 496 } 497 498 if (int_status & RK_MMU_IRQ_BUS_ERROR) 499 dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); 500 501 if (int_status & ~RK_MMU_IRQ_MASK) 502 dev_err(iommu->dev, "unexpected int_status: %#08x\n", 503 int_status); 504 505 rk_iommu_write(iommu, RK_MMU_INT_CLEAR, int_status); 506 507 return IRQ_HANDLED; 508} 509 510static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, 511 dma_addr_t iova) 512{ 513 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 514 unsigned long flags; 515 phys_addr_t pt_phys, phys = 0; 516 u32 dte, pte; 517 u32 *page_table; 518 519 spin_lock_irqsave(&rk_domain->dt_lock, flags); 520 521 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 522 if (!rk_dte_is_pt_valid(dte)) 523 goto out; 524 525 pt_phys = rk_dte_pt_address(dte); 526 page_table = (u32 *)phys_to_virt(pt_phys); 527 pte = page_table[rk_iova_pte_index(iova)]; 528 if (!rk_pte_is_page_valid(pte)) 529 goto out; 530 531 phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova); 532out: 533 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 534 535 return phys; 536} 537 538static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, 539 dma_addr_t iova, size_t size) 540{ 541 struct list_head *pos; 542 unsigned long flags; 543 544 /* shootdown these iova from all iommus using this domain */ 545 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 546 list_for_each(pos, &rk_domain->iommus) { 547 struct rk_iommu *iommu; 548 iommu = list_entry(pos, struct rk_iommu, node); 549 rk_iommu_zap_lines(iommu, iova, size); 550 } 551 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 552} 553 554static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 555 dma_addr_t iova) 556{ 557 u32 *page_table, *dte_addr; 558 u32 dte; 559 phys_addr_t pt_phys; 560 561 assert_spin_locked(&rk_domain->dt_lock); 562 563 dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; 564 dte = *dte_addr; 565 if (rk_dte_is_pt_valid(dte)) 566 goto done; 567 568 page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); 569 if (!page_table) 570 return ERR_PTR(-ENOMEM); 571 572 dte = rk_mk_dte(page_table); 573 *dte_addr = dte; 574 575 rk_table_flush(page_table, NUM_PT_ENTRIES); 576 rk_table_flush(dte_addr, 1); 577 578 /* 579 * Zap the first iova of newly allocated page table so iommu evicts 580 * old cached value of new dte from the iotlb. 581 */ 582 rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); 583 584done: 585 pt_phys = rk_dte_pt_address(dte); 586 return (u32 *)phys_to_virt(pt_phys); 587} 588 589static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 590 u32 *pte_addr, dma_addr_t iova, size_t size) 591{ 592 unsigned int pte_count; 593 unsigned int pte_total = size / SPAGE_SIZE; 594 595 assert_spin_locked(&rk_domain->dt_lock); 596 597 for (pte_count = 0; pte_count < pte_total; pte_count++) { 598 u32 pte = pte_addr[pte_count]; 599 if (!rk_pte_is_page_valid(pte)) 600 break; 601 602 pte_addr[pte_count] = rk_mk_pte_invalid(pte); 603 } 604 605 rk_table_flush(pte_addr, pte_count); 606 607 return pte_count * SPAGE_SIZE; 608} 609 610static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 611 dma_addr_t iova, phys_addr_t paddr, size_t size, 612 int prot) 613{ 614 unsigned int pte_count; 615 unsigned int pte_total = size / SPAGE_SIZE; 616 phys_addr_t page_phys; 617 618 assert_spin_locked(&rk_domain->dt_lock); 619 620 for (pte_count = 0; pte_count < pte_total; pte_count++) { 621 u32 pte = pte_addr[pte_count]; 622 623 if (rk_pte_is_page_valid(pte)) 624 goto unwind; 625 626 pte_addr[pte_count] = rk_mk_pte(paddr, prot); 627 628 paddr += SPAGE_SIZE; 629 } 630 631 rk_table_flush(pte_addr, pte_count); 632 633 return 0; 634unwind: 635 /* Unmap the range of iovas that we just mapped */ 636 rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); 637 638 iova += pte_count * SPAGE_SIZE; 639 page_phys = rk_pte_page_address(pte_addr[pte_count]); 640 pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", 641 &iova, &page_phys, &paddr, prot); 642 643 return -EADDRINUSE; 644} 645 646static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, 647 phys_addr_t paddr, size_t size, int prot) 648{ 649 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 650 unsigned long flags; 651 dma_addr_t iova = (dma_addr_t)_iova; 652 u32 *page_table, *pte_addr; 653 int ret; 654 655 spin_lock_irqsave(&rk_domain->dt_lock, flags); 656 657 /* 658 * pgsize_bitmap specifies iova sizes that fit in one page table 659 * (1024 4-KiB pages = 4 MiB). 660 * So, size will always be 4096 <= size <= 4194304. 661 * Since iommu_map() guarantees that both iova and size will be 662 * aligned, we will always only be mapping from a single dte here. 663 */ 664 page_table = rk_dte_get_page_table(rk_domain, iova); 665 if (IS_ERR(page_table)) { 666 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 667 return PTR_ERR(page_table); 668 } 669 670 pte_addr = &page_table[rk_iova_pte_index(iova)]; 671 ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); 672 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 673 674 return ret; 675} 676 677static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, 678 size_t size) 679{ 680 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 681 unsigned long flags; 682 dma_addr_t iova = (dma_addr_t)_iova; 683 phys_addr_t pt_phys; 684 u32 dte; 685 u32 *pte_addr; 686 size_t unmap_size; 687 688 spin_lock_irqsave(&rk_domain->dt_lock, flags); 689 690 /* 691 * pgsize_bitmap specifies iova sizes that fit in one page table 692 * (1024 4-KiB pages = 4 MiB). 693 * So, size will always be 4096 <= size <= 4194304. 694 * Since iommu_unmap() guarantees that both iova and size will be 695 * aligned, we will always only be unmapping from a single dte here. 696 */ 697 dte = rk_domain->dt[rk_iova_dte_index(iova)]; 698 /* Just return 0 if iova is unmapped */ 699 if (!rk_dte_is_pt_valid(dte)) { 700 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 701 return 0; 702 } 703 704 pt_phys = rk_dte_pt_address(dte); 705 pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 706 unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); 707 708 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 709 710 /* Shootdown iotlb entries for iova range that was just unmapped */ 711 rk_iommu_zap_iova(rk_domain, iova, unmap_size); 712 713 return unmap_size; 714} 715 716static struct rk_iommu *rk_iommu_from_dev(struct device *dev) 717{ 718 struct iommu_group *group; 719 struct device *iommu_dev; 720 struct rk_iommu *rk_iommu; 721 722 group = iommu_group_get(dev); 723 if (!group) 724 return NULL; 725 iommu_dev = iommu_group_get_iommudata(group); 726 rk_iommu = dev_get_drvdata(iommu_dev); 727 iommu_group_put(group); 728 729 return rk_iommu; 730} 731 732static int rk_iommu_attach_device(struct iommu_domain *domain, 733 struct device *dev) 734{ 735 struct rk_iommu *iommu; 736 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 737 unsigned long flags; 738 int ret; 739 phys_addr_t dte_addr; 740 741 /* 742 * Allow 'virtual devices' (e.g., drm) to attach to domain. 743 * Such a device does not belong to an iommu group. 744 */ 745 iommu = rk_iommu_from_dev(dev); 746 if (!iommu) 747 return 0; 748 749 ret = rk_iommu_enable_stall(iommu); 750 if (ret) 751 return ret; 752 753 ret = rk_iommu_force_reset(iommu); 754 if (ret) 755 return ret; 756 757 iommu->domain = domain; 758 759 ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, 760 IRQF_SHARED, dev_name(dev), iommu); 761 if (ret) 762 return ret; 763 764 dte_addr = virt_to_phys(rk_domain->dt); 765 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, dte_addr); 766 rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); 767 rk_iommu_write(iommu, RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 768 769 ret = rk_iommu_enable_paging(iommu); 770 if (ret) 771 return ret; 772 773 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 774 list_add_tail(&iommu->node, &rk_domain->iommus); 775 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 776 777 dev_info(dev, "Attached to iommu domain\n"); 778 779 rk_iommu_disable_stall(iommu); 780 781 return 0; 782} 783 784static void rk_iommu_detach_device(struct iommu_domain *domain, 785 struct device *dev) 786{ 787 struct rk_iommu *iommu; 788 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 789 unsigned long flags; 790 791 /* Allow 'virtual devices' (eg drm) to detach from domain */ 792 iommu = rk_iommu_from_dev(dev); 793 if (!iommu) 794 return; 795 796 spin_lock_irqsave(&rk_domain->iommus_lock, flags); 797 list_del_init(&iommu->node); 798 spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); 799 800 /* Ignore error while disabling, just keep going */ 801 rk_iommu_enable_stall(iommu); 802 rk_iommu_disable_paging(iommu); 803 rk_iommu_write(iommu, RK_MMU_INT_MASK, 0); 804 rk_iommu_write(iommu, RK_MMU_DTE_ADDR, 0); 805 rk_iommu_disable_stall(iommu); 806 807 devm_free_irq(dev, iommu->irq, iommu); 808 809 iommu->domain = NULL; 810 811 dev_info(dev, "Detached from iommu domain\n"); 812} 813 814static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 815{ 816 struct rk_iommu_domain *rk_domain; 817 818 if (type != IOMMU_DOMAIN_UNMANAGED) 819 return NULL; 820 821 rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); 822 if (!rk_domain) 823 return NULL; 824 825 /* 826 * rk32xx iommus use a 2 level pagetable. 827 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 828 * Allocate one 4 KiB page for each table. 829 */ 830 rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 831 if (!rk_domain->dt) 832 goto err_dt; 833 834 rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); 835 836 spin_lock_init(&rk_domain->iommus_lock); 837 spin_lock_init(&rk_domain->dt_lock); 838 INIT_LIST_HEAD(&rk_domain->iommus); 839 840 return &rk_domain->domain; 841 842err_dt: 843 kfree(rk_domain); 844 return NULL; 845} 846 847static void rk_iommu_domain_free(struct iommu_domain *domain) 848{ 849 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 850 int i; 851 852 WARN_ON(!list_empty(&rk_domain->iommus)); 853 854 for (i = 0; i < NUM_DT_ENTRIES; i++) { 855 u32 dte = rk_domain->dt[i]; 856 if (rk_dte_is_pt_valid(dte)) { 857 phys_addr_t pt_phys = rk_dte_pt_address(dte); 858 u32 *page_table = phys_to_virt(pt_phys); 859 free_page((unsigned long)page_table); 860 } 861 } 862 863 free_page((unsigned long)rk_domain->dt); 864 kfree(rk_domain); 865} 866 867static bool rk_iommu_is_dev_iommu_master(struct device *dev) 868{ 869 struct device_node *np = dev->of_node; 870 int ret; 871 872 /* 873 * An iommu master has an iommus property containing a list of phandles 874 * to iommu nodes, each with an #iommu-cells property with value 0. 875 */ 876 ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); 877 return (ret > 0); 878} 879 880static int rk_iommu_group_set_iommudata(struct iommu_group *group, 881 struct device *dev) 882{ 883 struct device_node *np = dev->of_node; 884 struct platform_device *pd; 885 int ret; 886 struct of_phandle_args args; 887 888 /* 889 * An iommu master has an iommus property containing a list of phandles 890 * to iommu nodes, each with an #iommu-cells property with value 0. 891 */ 892 ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, 893 &args); 894 if (ret) { 895 dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n", 896 np->full_name, ret); 897 return ret; 898 } 899 if (args.args_count != 0) { 900 dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n", 901 args.np->full_name, args.args_count); 902 return -EINVAL; 903 } 904 905 pd = of_find_device_by_node(args.np); 906 of_node_put(args.np); 907 if (!pd) { 908 dev_err(dev, "iommu %s not found\n", args.np->full_name); 909 return -EPROBE_DEFER; 910 } 911 912 /* TODO(djkurtz): handle multiple slave iommus for a single master */ 913 iommu_group_set_iommudata(group, &pd->dev, NULL); 914 915 return 0; 916} 917 918static int rk_iommu_add_device(struct device *dev) 919{ 920 struct iommu_group *group; 921 int ret; 922 923 if (!rk_iommu_is_dev_iommu_master(dev)) 924 return -ENODEV; 925 926 group = iommu_group_get(dev); 927 if (!group) { 928 group = iommu_group_alloc(); 929 if (IS_ERR(group)) { 930 dev_err(dev, "Failed to allocate IOMMU group\n"); 931 return PTR_ERR(group); 932 } 933 } 934 935 ret = iommu_group_add_device(group, dev); 936 if (ret) 937 goto err_put_group; 938 939 ret = rk_iommu_group_set_iommudata(group, dev); 940 if (ret) 941 goto err_remove_device; 942 943 iommu_group_put(group); 944 945 return 0; 946 947err_remove_device: 948 iommu_group_remove_device(dev); 949err_put_group: 950 iommu_group_put(group); 951 return ret; 952} 953 954static void rk_iommu_remove_device(struct device *dev) 955{ 956 if (!rk_iommu_is_dev_iommu_master(dev)) 957 return; 958 959 iommu_group_remove_device(dev); 960} 961 962static const struct iommu_ops rk_iommu_ops = { 963 .domain_alloc = rk_iommu_domain_alloc, 964 .domain_free = rk_iommu_domain_free, 965 .attach_dev = rk_iommu_attach_device, 966 .detach_dev = rk_iommu_detach_device, 967 .map = rk_iommu_map, 968 .unmap = rk_iommu_unmap, 969 .add_device = rk_iommu_add_device, 970 .remove_device = rk_iommu_remove_device, 971 .iova_to_phys = rk_iommu_iova_to_phys, 972 .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 973}; 974 975static int rk_iommu_probe(struct platform_device *pdev) 976{ 977 struct device *dev = &pdev->dev; 978 struct rk_iommu *iommu; 979 struct resource *res; 980 981 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 982 if (!iommu) 983 return -ENOMEM; 984 985 platform_set_drvdata(pdev, iommu); 986 iommu->dev = dev; 987 988 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 989 iommu->base = devm_ioremap_resource(&pdev->dev, res); 990 if (IS_ERR(iommu->base)) 991 return PTR_ERR(iommu->base); 992 993 iommu->irq = platform_get_irq(pdev, 0); 994 if (iommu->irq < 0) { 995 dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq); 996 return -ENXIO; 997 } 998 999 return 0; 1000} 1001 1002static int rk_iommu_remove(struct platform_device *pdev) 1003{ 1004 return 0; 1005} 1006 1007static const struct of_device_id rk_iommu_dt_ids[] = { 1008 { .compatible = "rockchip,iommu" }, 1009 { /* sentinel */ } 1010}; 1011MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); 1012 1013static struct platform_driver rk_iommu_driver = { 1014 .probe = rk_iommu_probe, 1015 .remove = rk_iommu_remove, 1016 .driver = { 1017 .name = "rk_iommu", 1018 .of_match_table = rk_iommu_dt_ids, 1019 }, 1020}; 1021 1022static int __init rk_iommu_init(void) 1023{ 1024 struct device_node *np; 1025 int ret; 1026 1027 np = of_find_matching_node(NULL, rk_iommu_dt_ids); 1028 if (!np) 1029 return 0; 1030 1031 of_node_put(np); 1032 1033 ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); 1034 if (ret) 1035 return ret; 1036 1037 return platform_driver_register(&rk_iommu_driver); 1038} 1039static void __exit rk_iommu_exit(void) 1040{ 1041 platform_driver_unregister(&rk_iommu_driver); 1042} 1043 1044subsys_initcall(rk_iommu_init); 1045module_exit(rk_iommu_exit); 1046 1047MODULE_DESCRIPTION("IOMMU API for Rockchip"); 1048MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>"); 1049MODULE_ALIAS("platform:rockchip-iommu"); 1050MODULE_LICENSE("GPL v2"); 1051