1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Intel MIC Host driver. 19 * 20 */ 21#include <linux/pci.h> 22 23#include "../common/mic_dev.h" 24#include "mic_device.h" 25#include "mic_smpt.h" 26 27static inline u64 mic_system_page_mask(struct mic_device *mdev) 28{ 29 return (1ULL << mdev->smpt->info.page_shift) - 1ULL; 30} 31 32static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa) 33{ 34 return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift; 35} 36 37static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index) 38{ 39 return mdev->smpt->info.base + (index * mdev->smpt->info.page_size); 40} 41 42static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa) 43{ 44 return pa & mic_system_page_mask(mdev); 45} 46 47static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa) 48{ 49 return ALIGN(pa - mic_system_page_mask(mdev), 50 mdev->smpt->info.page_size); 51} 52 53static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa) 54{ 55 return ALIGN(pa, mdev->smpt->info.page_size); 56} 57 58/* Total Cumulative system memory accessible by MIC across all SMPT entries */ 59static inline u64 mic_max_system_memory(struct mic_device *mdev) 60{ 61 return mdev->smpt->info.num_reg * mdev->smpt->info.page_size; 62} 63 64/* Maximum system memory address accessible by MIC */ 65static inline u64 mic_max_system_addr(struct mic_device *mdev) 66{ 67 return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL; 68} 69 70/* Check if the DMA address is a MIC system memory address */ 71static inline bool 72mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa) 73{ 74 return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev); 75} 76 77/* Populate an SMPT entry and update the reference counts. */ 78static void mic_add_smpt_entry(int spt, s64 *ref, u64 addr, 79 int entries, struct mic_device *mdev) 80{ 81 struct mic_smpt_info *smpt_info = mdev->smpt; 82 int i; 83 84 for (i = spt; i < spt + entries; i++, 85 addr += smpt_info->info.page_size) { 86 if (!smpt_info->entry[i].ref_count && 87 (smpt_info->entry[i].dma_addr != addr)) { 88 mdev->smpt_ops->set(mdev, addr, i); 89 smpt_info->entry[i].dma_addr = addr; 90 } 91 smpt_info->entry[i].ref_count += ref[i - spt]; 92 } 93} 94 95/* 96 * Find an available MIC address in MIC SMPT address space 97 * for a given DMA address and size. 98 */ 99static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr, 100 int entries, s64 *ref, size_t size) 101{ 102 int spt; 103 int ae = 0; 104 int i; 105 unsigned long flags; 106 dma_addr_t mic_addr = 0; 107 dma_addr_t addr = dma_addr; 108 struct mic_smpt_info *smpt_info = mdev->smpt; 109 110 spin_lock_irqsave(&smpt_info->smpt_lock, flags); 111 112 /* find existing entries */ 113 for (i = 0; i < smpt_info->info.num_reg; i++) { 114 if (smpt_info->entry[i].dma_addr == addr) { 115 ae++; 116 addr += smpt_info->info.page_size; 117 } else if (ae) /* cannot find contiguous entries */ 118 goto not_found; 119 120 if (ae == entries) 121 goto found; 122 } 123 124 /* find free entry */ 125 for (ae = 0, i = 0; i < smpt_info->info.num_reg; i++) { 126 ae = (smpt_info->entry[i].ref_count == 0) ? ae + 1 : 0; 127 if (ae == entries) 128 goto found; 129 } 130 131not_found: 132 spin_unlock_irqrestore(&smpt_info->smpt_lock, flags); 133 return mic_addr; 134 135found: 136 spt = i - entries + 1; 137 mic_addr = mic_smpt_to_pa(mdev, spt); 138 mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev); 139 smpt_info->map_count++; 140 smpt_info->ref_count += (s64)size; 141 spin_unlock_irqrestore(&smpt_info->smpt_lock, flags); 142 return mic_addr; 143} 144 145/* 146 * Returns number of smpt entries needed for dma_addr to dma_addr + size 147 * also returns the reference count array for each of those entries 148 * and the starting smpt address 149 */ 150static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr, 151 size_t size, s64 *ref, u64 *smpt_start) 152{ 153 u64 start = dma_addr; 154 u64 end = dma_addr + size; 155 int i = 0; 156 157 while (start < end) { 158 ref[i++] = min(mic_smpt_align_high(mdev, start + 1), 159 end) - start; 160 start = mic_smpt_align_high(mdev, start + 1); 161 } 162 163 if (smpt_start) 164 *smpt_start = mic_smpt_align_low(mdev, dma_addr); 165 166 return i; 167} 168 169/* 170 * mic_to_dma_addr - Converts a MIC address to a DMA address. 171 * 172 * @mdev: pointer to mic_device instance. 173 * @mic_addr: MIC address. 174 * 175 * returns a DMA address. 176 */ 177static dma_addr_t 178mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr) 179{ 180 struct mic_smpt_info *smpt_info = mdev->smpt; 181 int spt; 182 dma_addr_t dma_addr; 183 184 if (!mic_is_system_addr(mdev, mic_addr)) { 185 dev_err(mdev->sdev->parent, 186 "mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr); 187 return -EINVAL; 188 } 189 spt = mic_sys_addr_to_smpt(mdev, mic_addr); 190 dma_addr = smpt_info->entry[spt].dma_addr + 191 mic_smpt_offset(mdev, mic_addr); 192 return dma_addr; 193} 194 195/** 196 * mic_map - Maps a DMA address to a MIC physical address. 197 * 198 * @mdev: pointer to mic_device instance. 199 * @dma_addr: DMA address. 200 * @size: Size of the region to be mapped. 201 * 202 * This API converts the DMA address provided to a DMA address understood 203 * by MIC. Caller should check for errors by calling mic_map_error(..). 204 * 205 * returns DMA address as required by MIC. 206 */ 207dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size) 208{ 209 dma_addr_t mic_addr = 0; 210 int num_entries; 211 s64 *ref; 212 u64 smpt_start; 213 214 if (!size || size > mic_max_system_memory(mdev)) 215 return mic_addr; 216 217 ref = kmalloc(mdev->smpt->info.num_reg * sizeof(s64), GFP_KERNEL); 218 if (!ref) 219 return mic_addr; 220 221 num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size, 222 ref, &smpt_start); 223 224 /* Set the smpt table appropriately and get 16G aligned mic address */ 225 mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size); 226 227 kfree(ref); 228 229 /* 230 * If mic_addr is zero then its an error case 231 * since mic_addr can never be zero. 232 * else generate mic_addr by adding the 16G offset in dma_addr 233 */ 234 if (!mic_addr && MIC_FAMILY_X100 == mdev->family) { 235 dev_err(mdev->sdev->parent, 236 "mic_map failed dma_addr 0x%llx size 0x%lx\n", 237 dma_addr, size); 238 return mic_addr; 239 } else { 240 return mic_addr + mic_smpt_offset(mdev, dma_addr); 241 } 242} 243 244/** 245 * mic_unmap - Unmaps a MIC physical address. 246 * 247 * @mdev: pointer to mic_device instance. 248 * @mic_addr: MIC physical address. 249 * @size: Size of the region to be unmapped. 250 * 251 * This API unmaps the mappings created by mic_map(..). 252 * 253 * returns None. 254 */ 255void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size) 256{ 257 struct mic_smpt_info *smpt_info = mdev->smpt; 258 s64 *ref; 259 int num_smpt; 260 int spt; 261 int i; 262 unsigned long flags; 263 264 if (!size) 265 return; 266 267 if (!mic_is_system_addr(mdev, mic_addr)) { 268 dev_err(mdev->sdev->parent, 269 "invalid address: 0x%llx\n", mic_addr); 270 return; 271 } 272 273 spt = mic_sys_addr_to_smpt(mdev, mic_addr); 274 ref = kmalloc(mdev->smpt->info.num_reg * sizeof(s64), GFP_KERNEL); 275 if (!ref) 276 return; 277 278 /* Get number of smpt entries to be mapped, ref count array */ 279 num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL); 280 281 spin_lock_irqsave(&smpt_info->smpt_lock, flags); 282 smpt_info->unmap_count++; 283 smpt_info->ref_count -= (s64)size; 284 285 for (i = spt; i < spt + num_smpt; i++) { 286 smpt_info->entry[i].ref_count -= ref[i - spt]; 287 if (smpt_info->entry[i].ref_count < 0) 288 dev_warn(mdev->sdev->parent, 289 "ref count for entry %d is negative\n", i); 290 } 291 spin_unlock_irqrestore(&smpt_info->smpt_lock, flags); 292 kfree(ref); 293} 294 295/** 296 * mic_map_single - Maps a virtual address to a MIC physical address. 297 * 298 * @mdev: pointer to mic_device instance. 299 * @va: Kernel direct mapped virtual address. 300 * @size: Size of the region to be mapped. 301 * 302 * This API calls pci_map_single(..) for the direct mapped virtual address 303 * and then converts the DMA address provided to a DMA address understood 304 * by MIC. Caller should check for errors by calling mic_map_error(..). 305 * 306 * returns DMA address as required by MIC. 307 */ 308dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size) 309{ 310 dma_addr_t mic_addr = 0; 311 struct pci_dev *pdev = container_of(mdev->sdev->parent, 312 struct pci_dev, dev); 313 dma_addr_t dma_addr = 314 pci_map_single(pdev, va, size, PCI_DMA_BIDIRECTIONAL); 315 316 if (!pci_dma_mapping_error(pdev, dma_addr)) { 317 mic_addr = mic_map(mdev, dma_addr, size); 318 if (!mic_addr) { 319 dev_err(mdev->sdev->parent, 320 "mic_map failed dma_addr 0x%llx size 0x%lx\n", 321 dma_addr, size); 322 pci_unmap_single(pdev, dma_addr, 323 size, PCI_DMA_BIDIRECTIONAL); 324 } 325 } 326 return mic_addr; 327} 328 329/** 330 * mic_unmap_single - Unmaps a MIC physical address. 331 * 332 * @mdev: pointer to mic_device instance. 333 * @mic_addr: MIC physical address. 334 * @size: Size of the region to be unmapped. 335 * 336 * This API unmaps the mappings created by mic_map_single(..). 337 * 338 * returns None. 339 */ 340void 341mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size) 342{ 343 struct pci_dev *pdev = container_of(mdev->sdev->parent, 344 struct pci_dev, dev); 345 dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr); 346 mic_unmap(mdev, mic_addr, size); 347 pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); 348} 349 350/** 351 * mic_smpt_init - Initialize MIC System Memory Page Tables. 352 * 353 * @mdev: pointer to mic_device instance. 354 * 355 * returns 0 for success and -errno for error. 356 */ 357int mic_smpt_init(struct mic_device *mdev) 358{ 359 int i, err = 0; 360 dma_addr_t dma_addr; 361 struct mic_smpt_info *smpt_info; 362 363 mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL); 364 if (!mdev->smpt) 365 return -ENOMEM; 366 367 smpt_info = mdev->smpt; 368 mdev->smpt_ops->init(mdev); 369 smpt_info->entry = kmalloc_array(smpt_info->info.num_reg, 370 sizeof(*smpt_info->entry), GFP_KERNEL); 371 if (!smpt_info->entry) { 372 err = -ENOMEM; 373 goto free_smpt; 374 } 375 spin_lock_init(&smpt_info->smpt_lock); 376 for (i = 0; i < smpt_info->info.num_reg; i++) { 377 dma_addr = i * smpt_info->info.page_size; 378 smpt_info->entry[i].dma_addr = dma_addr; 379 smpt_info->entry[i].ref_count = 0; 380 mdev->smpt_ops->set(mdev, dma_addr, i); 381 } 382 smpt_info->ref_count = 0; 383 smpt_info->map_count = 0; 384 smpt_info->unmap_count = 0; 385 return 0; 386free_smpt: 387 kfree(smpt_info); 388 return err; 389} 390 391/** 392 * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables. 393 * 394 * @mdev: pointer to mic_device instance. 395 * 396 * returns None. 397 */ 398void mic_smpt_uninit(struct mic_device *mdev) 399{ 400 struct mic_smpt_info *smpt_info = mdev->smpt; 401 int i; 402 403 dev_dbg(mdev->sdev->parent, 404 "nodeid %d SMPT ref count %lld map %lld unmap %lld\n", 405 mdev->id, smpt_info->ref_count, 406 smpt_info->map_count, smpt_info->unmap_count); 407 408 for (i = 0; i < smpt_info->info.num_reg; i++) { 409 dev_dbg(mdev->sdev->parent, 410 "SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n", 411 i, smpt_info->entry[i].dma_addr, 412 smpt_info->entry[i].ref_count); 413 if (smpt_info->entry[i].ref_count) 414 dev_warn(mdev->sdev->parent, 415 "ref count for entry %d is not zero\n", i); 416 } 417 kfree(smpt_info->entry); 418 kfree(smpt_info); 419} 420 421/** 422 * mic_smpt_restore - Restore MIC System Memory Page Tables. 423 * 424 * @mdev: pointer to mic_device instance. 425 * 426 * Restore the SMPT registers to values previously stored in the 427 * SW data structures. Some MIC steppings lose register state 428 * across resets and this API should be called for performing 429 * a restore operation if required. 430 * 431 * returns None. 432 */ 433void mic_smpt_restore(struct mic_device *mdev) 434{ 435 int i; 436 dma_addr_t dma_addr; 437 438 for (i = 0; i < mdev->smpt->info.num_reg; i++) { 439 dma_addr = mdev->smpt->entry[i].dma_addr; 440 mdev->smpt_ops->set(mdev, dma_addr, i); 441 } 442} 443