1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Intel MIC Host driver. 19 * 20 */ 21#include <linux/pci.h> 22 23#include "../common/mic_dev.h" 24#include "mic_device.h" 25#include "mic_smpt.h" 26 27static inline u64 mic_system_page_mask(struct mic_device *mdev) 28{ 29 return (1ULL << mdev->smpt->info.page_shift) - 1ULL; 30} 31 32static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa) 33{ 34 return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift; 35} 36 37static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index) 38{ 39 return mdev->smpt->info.base + (index * mdev->smpt->info.page_size); 40} 41 42static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa) 43{ 44 return pa & mic_system_page_mask(mdev); 45} 46 47static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa) 48{ 49 return ALIGN(pa - mic_system_page_mask(mdev), 50 mdev->smpt->info.page_size); 51} 52 53static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa) 54{ 55 return ALIGN(pa, mdev->smpt->info.page_size); 56} 57 58/* Total Cumulative system memory accessible by MIC across all SMPT entries */ 59static inline u64 mic_max_system_memory(struct mic_device *mdev) 60{ 61 return mdev->smpt->info.num_reg * mdev->smpt->info.page_size; 62} 63 64/* Maximum system memory address accessible by MIC */ 65static inline u64 mic_max_system_addr(struct mic_device *mdev) 66{ 67 return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL; 68} 69 70/* Check if the DMA address is a MIC system memory address */ 71static inline bool 72mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa) 73{ 74 return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev); 75} 76 77/* Populate an SMPT entry and update the reference counts. */ 78static void mic_add_smpt_entry(int spt, s64 *ref, u64 addr, 79 int entries, struct mic_device *mdev) 80{ 81 struct mic_smpt_info *smpt_info = mdev->smpt; 82 int i; 83 84 for (i = spt; i < spt + entries; i++, 85 addr += smpt_info->info.page_size) { 86 if (!smpt_info->entry[i].ref_count && 87 (smpt_info->entry[i].dma_addr != addr)) { 88 mdev->smpt_ops->set(mdev, addr, i); 89 smpt_info->entry[i].dma_addr = addr; 90 } 91 smpt_info->entry[i].ref_count += ref[i - spt]; 92 } 93} 94 95/* 96 * Find an available MIC address in MIC SMPT address space 97 * for a given DMA address and size. 98 */ 99static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr, 100 int entries, s64 *ref, size_t size) 101{ 102 int spt; 103 int ae = 0; 104 int i; 105 unsigned long flags; 106 dma_addr_t mic_addr = 0; 107 dma_addr_t addr = dma_addr; 108 struct mic_smpt_info *smpt_info = mdev->smpt; 109 110 spin_lock_irqsave(&smpt_info->smpt_lock, flags); 111 112 /* find existing entries */ 113 for (i = 0; i < smpt_info->info.num_reg; i++) { 114 if (smpt_info->entry[i].dma_addr == addr) { 115 ae++; 116 addr += smpt_info->info.page_size; 117 } else if (ae) /* cannot find contiguous entries */ 118 goto not_found; 119 120 if (ae == entries) 121 goto found; 122 } 123 124 /* find free entry */ 125 for (ae = 0, i = 0; i < smpt_info->info.num_reg; i++) { 126 ae = (smpt_info->entry[i].ref_count == 0) ? ae + 1 : 0; 127 if (ae == entries) 128 goto found; 129 } 130 131not_found: 132 spin_unlock_irqrestore(&smpt_info->smpt_lock, flags); 133 return mic_addr; 134 135found: 136 spt = i - entries + 1; 137 mic_addr = mic_smpt_to_pa(mdev, spt); 138 mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev); 139 smpt_info->map_count++; 140 smpt_info->ref_count += (s64)size; 141 spin_unlock_irqrestore(&smpt_info->smpt_lock, flags); 142 return mic_addr; 143} 144 145/* 146 * Returns number of smpt entries needed for dma_addr to dma_addr + size 147 * also returns the reference count array for each of those entries 148 * and the starting smpt address 149 */ 150static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr, 151 size_t size, s64 *ref, u64 *smpt_start) 152{ 153 u64 start = dma_addr; 154 u64 end = dma_addr + size; 155 int i = 0; 156 157 while (start < end) { 158 ref[i++] = min(mic_smpt_align_high(mdev, start + 1), 159 end) - start; 160 start = mic_smpt_align_high(mdev, start + 1); 161 } 162 163 if (smpt_start) 164 *smpt_start = mic_smpt_align_low(mdev, dma_addr); 165 166 return i; 167} 168 169/* 170 * mic_to_dma_addr - Converts a MIC address to a DMA address. 171 * 172 * @mdev: pointer to mic_device instance. 173 * @mic_addr: MIC address. 174 * 175 * returns a DMA address. 176 */ 177dma_addr_t mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr) 178{ 179 struct mic_smpt_info *smpt_info = mdev->smpt; 180 int spt; 181 dma_addr_t dma_addr; 182 183 if (!mic_is_system_addr(mdev, mic_addr)) { 184 dev_err(&mdev->pdev->dev, 185 "mic_addr is invalid. mic_addr = 0x%llx\n", mic_addr); 186 return -EINVAL; 187 } 188 spt = mic_sys_addr_to_smpt(mdev, mic_addr); 189 dma_addr = smpt_info->entry[spt].dma_addr + 190 mic_smpt_offset(mdev, mic_addr); 191 return dma_addr; 192} 193 194/** 195 * mic_map - Maps a DMA address to a MIC physical address. 196 * 197 * @mdev: pointer to mic_device instance. 198 * @dma_addr: DMA address. 199 * @size: Size of the region to be mapped. 200 * 201 * This API converts the DMA address provided to a DMA address understood 202 * by MIC. Caller should check for errors by calling mic_map_error(..). 203 * 204 * returns DMA address as required by MIC. 205 */ 206dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size) 207{ 208 dma_addr_t mic_addr = 0; 209 int num_entries; 210 s64 *ref; 211 u64 smpt_start; 212 213 if (!size || size > mic_max_system_memory(mdev)) 214 return mic_addr; 215 216 ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC); 217 if (!ref) 218 return mic_addr; 219 220 num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size, 221 ref, &smpt_start); 222 223 /* Set the smpt table appropriately and get 16G aligned mic address */ 224 mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size); 225 226 kfree(ref); 227 228 /* 229 * If mic_addr is zero then its an error case 230 * since mic_addr can never be zero. 231 * else generate mic_addr by adding the 16G offset in dma_addr 232 */ 233 if (!mic_addr && MIC_FAMILY_X100 == mdev->family) { 234 dev_err(&mdev->pdev->dev, 235 "mic_map failed dma_addr 0x%llx size 0x%lx\n", 236 dma_addr, size); 237 return mic_addr; 238 } else { 239 return mic_addr + mic_smpt_offset(mdev, dma_addr); 240 } 241} 242 243/** 244 * mic_unmap - Unmaps a MIC physical address. 245 * 246 * @mdev: pointer to mic_device instance. 247 * @mic_addr: MIC physical address. 248 * @size: Size of the region to be unmapped. 249 * 250 * This API unmaps the mappings created by mic_map(..). 251 * 252 * returns None. 253 */ 254void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size) 255{ 256 struct mic_smpt_info *smpt_info = mdev->smpt; 257 s64 *ref; 258 int num_smpt; 259 int spt; 260 int i; 261 unsigned long flags; 262 263 if (!size) 264 return; 265 266 if (!mic_is_system_addr(mdev, mic_addr)) { 267 dev_err(&mdev->pdev->dev, 268 "invalid address: 0x%llx\n", mic_addr); 269 return; 270 } 271 272 spt = mic_sys_addr_to_smpt(mdev, mic_addr); 273 ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC); 274 if (!ref) 275 return; 276 277 /* Get number of smpt entries to be mapped, ref count array */ 278 num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL); 279 280 spin_lock_irqsave(&smpt_info->smpt_lock, flags); 281 smpt_info->unmap_count++; 282 smpt_info->ref_count -= (s64)size; 283 284 for (i = spt; i < spt + num_smpt; i++) { 285 smpt_info->entry[i].ref_count -= ref[i - spt]; 286 if (smpt_info->entry[i].ref_count < 0) 287 dev_warn(&mdev->pdev->dev, 288 "ref count for entry %d is negative\n", i); 289 } 290 spin_unlock_irqrestore(&smpt_info->smpt_lock, flags); 291 kfree(ref); 292} 293 294/** 295 * mic_map_single - Maps a virtual address to a MIC physical address. 296 * 297 * @mdev: pointer to mic_device instance. 298 * @va: Kernel direct mapped virtual address. 299 * @size: Size of the region to be mapped. 300 * 301 * This API calls pci_map_single(..) for the direct mapped virtual address 302 * and then converts the DMA address provided to a DMA address understood 303 * by MIC. Caller should check for errors by calling mic_map_error(..). 304 * 305 * returns DMA address as required by MIC. 306 */ 307dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size) 308{ 309 dma_addr_t mic_addr = 0; 310 struct pci_dev *pdev = mdev->pdev; 311 dma_addr_t dma_addr = 312 pci_map_single(pdev, va, size, PCI_DMA_BIDIRECTIONAL); 313 314 if (!pci_dma_mapping_error(pdev, dma_addr)) { 315 mic_addr = mic_map(mdev, dma_addr, size); 316 if (!mic_addr) { 317 dev_err(&mdev->pdev->dev, 318 "mic_map failed dma_addr 0x%llx size 0x%lx\n", 319 dma_addr, size); 320 pci_unmap_single(pdev, dma_addr, 321 size, PCI_DMA_BIDIRECTIONAL); 322 } 323 } 324 return mic_addr; 325} 326 327/** 328 * mic_unmap_single - Unmaps a MIC physical address. 329 * 330 * @mdev: pointer to mic_device instance. 331 * @mic_addr: MIC physical address. 332 * @size: Size of the region to be unmapped. 333 * 334 * This API unmaps the mappings created by mic_map_single(..). 335 * 336 * returns None. 337 */ 338void 339mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size) 340{ 341 struct pci_dev *pdev = mdev->pdev; 342 dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr); 343 mic_unmap(mdev, mic_addr, size); 344 pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); 345} 346 347/** 348 * mic_smpt_init - Initialize MIC System Memory Page Tables. 349 * 350 * @mdev: pointer to mic_device instance. 351 * 352 * returns 0 for success and -errno for error. 353 */ 354int mic_smpt_init(struct mic_device *mdev) 355{ 356 int i, err = 0; 357 dma_addr_t dma_addr; 358 struct mic_smpt_info *smpt_info; 359 360 mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL); 361 if (!mdev->smpt) 362 return -ENOMEM; 363 364 smpt_info = mdev->smpt; 365 mdev->smpt_ops->init(mdev); 366 smpt_info->entry = kmalloc_array(smpt_info->info.num_reg, 367 sizeof(*smpt_info->entry), GFP_KERNEL); 368 if (!smpt_info->entry) { 369 err = -ENOMEM; 370 goto free_smpt; 371 } 372 spin_lock_init(&smpt_info->smpt_lock); 373 for (i = 0; i < smpt_info->info.num_reg; i++) { 374 dma_addr = i * smpt_info->info.page_size; 375 smpt_info->entry[i].dma_addr = dma_addr; 376 smpt_info->entry[i].ref_count = 0; 377 mdev->smpt_ops->set(mdev, dma_addr, i); 378 } 379 smpt_info->ref_count = 0; 380 smpt_info->map_count = 0; 381 smpt_info->unmap_count = 0; 382 return 0; 383free_smpt: 384 kfree(smpt_info); 385 return err; 386} 387 388/** 389 * mic_smpt_uninit - UnInitialize MIC System Memory Page Tables. 390 * 391 * @mdev: pointer to mic_device instance. 392 * 393 * returns None. 394 */ 395void mic_smpt_uninit(struct mic_device *mdev) 396{ 397 struct mic_smpt_info *smpt_info = mdev->smpt; 398 int i; 399 400 dev_dbg(&mdev->pdev->dev, 401 "nodeid %d SMPT ref count %lld map %lld unmap %lld\n", 402 mdev->id, smpt_info->ref_count, 403 smpt_info->map_count, smpt_info->unmap_count); 404 405 for (i = 0; i < smpt_info->info.num_reg; i++) { 406 dev_dbg(&mdev->pdev->dev, 407 "SMPT entry[%d] dma_addr = 0x%llx ref_count = %lld\n", 408 i, smpt_info->entry[i].dma_addr, 409 smpt_info->entry[i].ref_count); 410 if (smpt_info->entry[i].ref_count) 411 dev_warn(&mdev->pdev->dev, 412 "ref count for entry %d is not zero\n", i); 413 } 414 kfree(smpt_info->entry); 415 kfree(smpt_info); 416} 417 418/** 419 * mic_smpt_restore - Restore MIC System Memory Page Tables. 420 * 421 * @mdev: pointer to mic_device instance. 422 * 423 * Restore the SMPT registers to values previously stored in the 424 * SW data structures. Some MIC steppings lose register state 425 * across resets and this API should be called for performing 426 * a restore operation if required. 427 * 428 * returns None. 429 */ 430void mic_smpt_restore(struct mic_device *mdev) 431{ 432 int i; 433 dma_addr_t dma_addr; 434 435 for (i = 0; i < mdev->smpt->info.num_reg; i++) { 436 dma_addr = mdev->smpt->entry[i].dma_addr; 437 mdev->smpt_ops->set(mdev, dma_addr, i); 438 } 439} 440