1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Intel MIC Host driver. 19 * 20 */ 21#include <linux/fs.h> 22#include <linux/pci.h> 23#include <linux/sched.h> 24#include <linux/firmware.h> 25#include <linux/delay.h> 26 27#include "../common/mic_dev.h" 28#include "mic_device.h" 29#include "mic_x100.h" 30#include "mic_smpt.h" 31 32/** 33 * mic_x100_write_spad - write to the scratchpad register 34 * @mdev: pointer to mic_device instance 35 * @idx: index to the scratchpad register, 0 based 36 * @val: the data value to put into the register 37 * 38 * This function allows writing of a 32bit value to the indexed scratchpad 39 * register. 40 * 41 * RETURNS: none. 42 */ 43static void 44mic_x100_write_spad(struct mic_device *mdev, unsigned int idx, u32 val) 45{ 46 dev_dbg(mdev->sdev->parent, "Writing 0x%x to scratch pad index %d\n", 47 val, idx); 48 mic_mmio_write(&mdev->mmio, val, 49 MIC_X100_SBOX_BASE_ADDRESS + 50 MIC_X100_SBOX_SPAD0 + idx * 4); 51} 52 53/** 54 * mic_x100_read_spad - read from the scratchpad register 55 * @mdev: pointer to mic_device instance 56 * @idx: index to scratchpad register, 0 based 57 * 58 * This function allows reading of the 32bit scratchpad register. 59 * 60 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 61 */ 62static u32 63mic_x100_read_spad(struct mic_device *mdev, unsigned int idx) 64{ 65 u32 val = mic_mmio_read(&mdev->mmio, 66 MIC_X100_SBOX_BASE_ADDRESS + 67 MIC_X100_SBOX_SPAD0 + idx * 4); 68 69 dev_dbg(mdev->sdev->parent, 70 "Reading 0x%x from scratch pad index %d\n", val, idx); 71 return val; 72} 73 74/** 75 * mic_x100_enable_interrupts - Enable interrupts. 76 * @mdev: pointer to mic_device instance 77 */ 78static void mic_x100_enable_interrupts(struct mic_device *mdev) 79{ 80 u32 reg; 81 struct mic_mw *mw = &mdev->mmio; 82 u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0; 83 u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0; 84 85 reg = mic_mmio_read(mw, sice0); 86 reg |= MIC_X100_SBOX_DBR_BITS(0xf) | MIC_X100_SBOX_DMA_BITS(0xff); 87 mic_mmio_write(mw, reg, sice0); 88 89 /* 90 * Enable auto-clear when enabling interrupts. Applicable only for 91 * MSI-x. Legacy and MSI mode cannot have auto-clear enabled. 92 */ 93 if (mdev->irq_info.num_vectors > 1) { 94 reg = mic_mmio_read(mw, siac0); 95 reg |= MIC_X100_SBOX_DBR_BITS(0xf) | 96 MIC_X100_SBOX_DMA_BITS(0xff); 97 mic_mmio_write(mw, reg, siac0); 98 } 99} 100 101/** 102 * mic_x100_disable_interrupts - Disable interrupts. 103 * @mdev: pointer to mic_device instance 104 */ 105static void mic_x100_disable_interrupts(struct mic_device *mdev) 106{ 107 u32 reg; 108 struct mic_mw *mw = &mdev->mmio; 109 u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0; 110 u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0; 111 u32 sicc0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICC0; 112 113 reg = mic_mmio_read(mw, sice0); 114 mic_mmio_write(mw, reg, sicc0); 115 116 if (mdev->irq_info.num_vectors > 1) { 117 reg = mic_mmio_read(mw, siac0); 118 reg &= ~(MIC_X100_SBOX_DBR_BITS(0xf) | 119 MIC_X100_SBOX_DMA_BITS(0xff)); 120 mic_mmio_write(mw, reg, siac0); 121 } 122} 123 124/** 125 * mic_x100_send_sbox_intr - Send an MIC_X100_SBOX interrupt to MIC. 126 * @mdev: pointer to mic_device instance 127 */ 128static void mic_x100_send_sbox_intr(struct mic_device *mdev, 129 int doorbell) 130{ 131 struct mic_mw *mw = &mdev->mmio; 132 u64 apic_icr_offset = MIC_X100_SBOX_APICICR0 + doorbell * 8; 133 u32 apicicr_low = mic_mmio_read(mw, MIC_X100_SBOX_BASE_ADDRESS + 134 apic_icr_offset); 135 136 /* for MIC we need to make sure we "hit" the send_icr bit (13) */ 137 apicicr_low = (apicicr_low | (1 << 13)); 138 139 /* Ensure that the interrupt is ordered w.r.t. previous stores. */ 140 wmb(); 141 mic_mmio_write(mw, apicicr_low, 142 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset); 143} 144 145/** 146 * mic_x100_send_rdmasr_intr - Send an RDMASR interrupt to MIC. 147 * @mdev: pointer to mic_device instance 148 */ 149static void mic_x100_send_rdmasr_intr(struct mic_device *mdev, 150 int doorbell) 151{ 152 int rdmasr_offset = MIC_X100_SBOX_RDMASR0 + (doorbell << 2); 153 /* Ensure that the interrupt is ordered w.r.t. previous stores. */ 154 wmb(); 155 mic_mmio_write(&mdev->mmio, 0, 156 MIC_X100_SBOX_BASE_ADDRESS + rdmasr_offset); 157} 158 159/** 160 * __mic_x100_send_intr - Send interrupt to MIC. 161 * @mdev: pointer to mic_device instance 162 * @doorbell: doorbell number. 163 */ 164static void mic_x100_send_intr(struct mic_device *mdev, int doorbell) 165{ 166 int rdmasr_db; 167 if (doorbell < MIC_X100_NUM_SBOX_IRQ) { 168 mic_x100_send_sbox_intr(mdev, doorbell); 169 } else { 170 rdmasr_db = doorbell - MIC_X100_NUM_SBOX_IRQ + 171 MIC_X100_RDMASR_IRQ_BASE; 172 mic_x100_send_rdmasr_intr(mdev, rdmasr_db); 173 } 174} 175 176/** 177 * mic_x100_ack_interrupt - Read the interrupt sources register and 178 * clear it. This function will be called in the MSI/INTx case. 179 * @mdev: Pointer to mic_device instance. 180 * 181 * Returns: bitmask of interrupt sources triggered. 182 */ 183static u32 mic_x100_ack_interrupt(struct mic_device *mdev) 184{ 185 u32 sicr0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICR0; 186 u32 reg = mic_mmio_read(&mdev->mmio, sicr0); 187 mic_mmio_write(&mdev->mmio, reg, sicr0); 188 return reg; 189} 190 191/** 192 * mic_x100_intr_workarounds - These hardware specific workarounds are 193 * to be invoked everytime an interrupt is handled. 194 * @mdev: Pointer to mic_device instance. 195 * 196 * Returns: none 197 */ 198static void mic_x100_intr_workarounds(struct mic_device *mdev) 199{ 200 struct mic_mw *mw = &mdev->mmio; 201 202 /* Clear pending bit array. */ 203 if (MIC_A0_STEP == mdev->stepping) 204 mic_mmio_write(mw, 1, MIC_X100_SBOX_BASE_ADDRESS + 205 MIC_X100_SBOX_MSIXPBACR); 206 207 if (mdev->stepping >= MIC_B0_STEP) 208 mdev->intr_ops->enable_interrupts(mdev); 209} 210 211/** 212 * mic_x100_hw_intr_init - Initialize h/w specific interrupt 213 * information. 214 * @mdev: pointer to mic_device instance 215 */ 216static void mic_x100_hw_intr_init(struct mic_device *mdev) 217{ 218 mdev->intr_info = (struct mic_intr_info *)mic_x100_intr_init; 219} 220 221/** 222 * mic_x100_read_msi_to_src_map - read from the MSI mapping registers 223 * @mdev: pointer to mic_device instance 224 * @idx: index to the mapping register, 0 based 225 * 226 * This function allows reading of the 32bit MSI mapping register. 227 * 228 * RETURNS: The value in the register. 229 */ 230static u32 231mic_x100_read_msi_to_src_map(struct mic_device *mdev, int idx) 232{ 233 return mic_mmio_read(&mdev->mmio, 234 MIC_X100_SBOX_BASE_ADDRESS + 235 MIC_X100_SBOX_MXAR0 + idx * 4); 236} 237 238/** 239 * mic_x100_program_msi_to_src_map - program the MSI mapping registers 240 * @mdev: pointer to mic_device instance 241 * @idx: index to the mapping register, 0 based 242 * @offset: The bit offset in the register that needs to be updated. 243 * @set: boolean specifying if the bit in the specified offset needs 244 * to be set or cleared. 245 * 246 * RETURNS: None. 247 */ 248static void 249mic_x100_program_msi_to_src_map(struct mic_device *mdev, 250 int idx, int offset, bool set) 251{ 252 unsigned long reg; 253 struct mic_mw *mw = &mdev->mmio; 254 u32 mxar = MIC_X100_SBOX_BASE_ADDRESS + 255 MIC_X100_SBOX_MXAR0 + idx * 4; 256 257 reg = mic_mmio_read(mw, mxar); 258 if (set) 259 __set_bit(offset, ®); 260 else 261 __clear_bit(offset, ®); 262 mic_mmio_write(mw, reg, mxar); 263} 264 265/* 266 * mic_x100_reset_fw_ready - Reset Firmware ready status field. 267 * @mdev: pointer to mic_device instance 268 */ 269static void mic_x100_reset_fw_ready(struct mic_device *mdev) 270{ 271 mdev->ops->write_spad(mdev, MIC_X100_DOWNLOAD_INFO, 0); 272} 273 274/* 275 * mic_x100_is_fw_ready - Check if firmware is ready. 276 * @mdev: pointer to mic_device instance 277 */ 278static bool mic_x100_is_fw_ready(struct mic_device *mdev) 279{ 280 u32 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO); 281 return MIC_X100_SPAD2_DOWNLOAD_STATUS(scratch2) ? true : false; 282} 283 284/** 285 * mic_x100_get_apic_id - Get bootstrap APIC ID. 286 * @mdev: pointer to mic_device instance 287 */ 288static u32 mic_x100_get_apic_id(struct mic_device *mdev) 289{ 290 u32 scratch2 = 0; 291 292 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO); 293 return MIC_X100_SPAD2_APIC_ID(scratch2); 294} 295 296/** 297 * mic_x100_send_firmware_intr - Send an interrupt to the firmware on MIC. 298 * @mdev: pointer to mic_device instance 299 */ 300static void mic_x100_send_firmware_intr(struct mic_device *mdev) 301{ 302 u32 apicicr_low; 303 u64 apic_icr_offset = MIC_X100_SBOX_APICICR7; 304 int vector = MIC_X100_BSP_INTERRUPT_VECTOR; 305 struct mic_mw *mw = &mdev->mmio; 306 307 /* 308 * For MIC we need to make sure we "hit" 309 * the send_icr bit (13). 310 */ 311 apicicr_low = (vector | (1 << 13)); 312 313 mic_mmio_write(mw, mic_x100_get_apic_id(mdev), 314 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset + 4); 315 316 /* Ensure that the interrupt is ordered w.r.t. previous stores. */ 317 wmb(); 318 mic_mmio_write(mw, apicicr_low, 319 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset); 320} 321 322/** 323 * mic_x100_hw_reset - Reset the MIC device. 324 * @mdev: pointer to mic_device instance 325 */ 326static void mic_x100_hw_reset(struct mic_device *mdev) 327{ 328 u32 reset_reg; 329 u32 rgcr = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_RGCR; 330 struct mic_mw *mw = &mdev->mmio; 331 332 /* Ensure that the reset is ordered w.r.t. previous loads and stores */ 333 mb(); 334 /* Trigger reset */ 335 reset_reg = mic_mmio_read(mw, rgcr); 336 reset_reg |= 0x1; 337 mic_mmio_write(mw, reset_reg, rgcr); 338 /* 339 * It seems we really want to delay at least 1 second 340 * after touching reset to prevent a lot of problems. 341 */ 342 msleep(1000); 343} 344 345/** 346 * mic_x100_load_command_line - Load command line to MIC. 347 * @mdev: pointer to mic_device instance 348 * @fw: the firmware image 349 * 350 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 351 */ 352static int 353mic_x100_load_command_line(struct mic_device *mdev, const struct firmware *fw) 354{ 355 u32 len = 0; 356 u32 boot_mem; 357 char *buf; 358 void __iomem *cmd_line_va = mdev->aper.va + mdev->bootaddr + fw->size; 359#define CMDLINE_SIZE 2048 360 361 boot_mem = mdev->aper.len >> 20; 362 buf = kzalloc(CMDLINE_SIZE, GFP_KERNEL); 363 if (!buf) { 364 dev_err(mdev->sdev->parent, 365 "%s %d allocation failed\n", __func__, __LINE__); 366 return -ENOMEM; 367 } 368 len += snprintf(buf, CMDLINE_SIZE - len, 369 " mem=%dM", boot_mem); 370 if (mdev->cmdline) 371 snprintf(buf + len, CMDLINE_SIZE - len, " %s", mdev->cmdline); 372 memcpy_toio(cmd_line_va, buf, strlen(buf) + 1); 373 kfree(buf); 374 return 0; 375} 376 377/** 378 * mic_x100_load_ramdisk - Load ramdisk to MIC. 379 * @mdev: pointer to mic_device instance 380 * 381 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 382 */ 383static int 384mic_x100_load_ramdisk(struct mic_device *mdev) 385{ 386 const struct firmware *fw; 387 int rc; 388 struct boot_params __iomem *bp = mdev->aper.va + mdev->bootaddr; 389 390 rc = request_firmware(&fw, 391 mdev->ramdisk, mdev->sdev->parent); 392 if (rc < 0) { 393 dev_err(mdev->sdev->parent, 394 "ramdisk request_firmware failed: %d %s\n", 395 rc, mdev->ramdisk); 396 goto error; 397 } 398 /* 399 * Typically the bootaddr for card OS is 64M 400 * so copy over the ramdisk @ 128M. 401 */ 402 memcpy_toio(mdev->aper.va + (mdev->bootaddr << 1), fw->data, fw->size); 403 iowrite32(mdev->bootaddr << 1, &bp->hdr.ramdisk_image); 404 iowrite32(fw->size, &bp->hdr.ramdisk_size); 405 release_firmware(fw); 406error: 407 return rc; 408} 409 410/** 411 * mic_x100_get_boot_addr - Get MIC boot address. 412 * @mdev: pointer to mic_device instance 413 * 414 * This function is called during firmware load to determine 415 * the address at which the OS should be downloaded in card 416 * memory i.e. GDDR. 417 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 418 */ 419static int 420mic_x100_get_boot_addr(struct mic_device *mdev) 421{ 422 u32 scratch2, boot_addr; 423 int rc = 0; 424 425 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO); 426 boot_addr = MIC_X100_SPAD2_DOWNLOAD_ADDR(scratch2); 427 dev_dbg(mdev->sdev->parent, "%s %d boot_addr 0x%x\n", 428 __func__, __LINE__, boot_addr); 429 if (boot_addr > (1 << 31)) { 430 dev_err(mdev->sdev->parent, 431 "incorrect bootaddr 0x%x\n", 432 boot_addr); 433 rc = -EINVAL; 434 goto error; 435 } 436 mdev->bootaddr = boot_addr; 437error: 438 return rc; 439} 440 441/** 442 * mic_x100_load_firmware - Load firmware to MIC. 443 * @mdev: pointer to mic_device instance 444 * @buf: buffer containing boot string including firmware/ramdisk path. 445 * 446 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 447 */ 448static int 449mic_x100_load_firmware(struct mic_device *mdev, const char *buf) 450{ 451 int rc; 452 const struct firmware *fw; 453 454 rc = mic_x100_get_boot_addr(mdev); 455 if (rc) 456 goto error; 457 /* load OS */ 458 rc = request_firmware(&fw, mdev->firmware, mdev->sdev->parent); 459 if (rc < 0) { 460 dev_err(mdev->sdev->parent, 461 "ramdisk request_firmware failed: %d %s\n", 462 rc, mdev->firmware); 463 goto error; 464 } 465 if (mdev->bootaddr > mdev->aper.len - fw->size) { 466 rc = -EINVAL; 467 dev_err(mdev->sdev->parent, "%s %d rc %d bootaddr 0x%x\n", 468 __func__, __LINE__, rc, mdev->bootaddr); 469 release_firmware(fw); 470 goto error; 471 } 472 memcpy_toio(mdev->aper.va + mdev->bootaddr, fw->data, fw->size); 473 mdev->ops->write_spad(mdev, MIC_X100_FW_SIZE, fw->size); 474 if (!strcmp(mdev->bootmode, "elf")) 475 goto done; 476 /* load command line */ 477 rc = mic_x100_load_command_line(mdev, fw); 478 if (rc) { 479 dev_err(mdev->sdev->parent, "%s %d rc %d\n", 480 __func__, __LINE__, rc); 481 goto error; 482 } 483 release_firmware(fw); 484 /* load ramdisk */ 485 if (mdev->ramdisk) 486 rc = mic_x100_load_ramdisk(mdev); 487error: 488 dev_dbg(mdev->sdev->parent, "%s %d rc %d\n", __func__, __LINE__, rc); 489done: 490 return rc; 491} 492 493/** 494 * mic_x100_get_postcode - Get postcode status from firmware. 495 * @mdev: pointer to mic_device instance 496 * 497 * RETURNS: postcode. 498 */ 499static u32 mic_x100_get_postcode(struct mic_device *mdev) 500{ 501 return mic_mmio_read(&mdev->mmio, MIC_X100_POSTCODE); 502} 503 504/** 505 * mic_x100_smpt_set - Update an SMPT entry with a DMA address. 506 * @mdev: pointer to mic_device instance 507 * 508 * RETURNS: none. 509 */ 510static void 511mic_x100_smpt_set(struct mic_device *mdev, dma_addr_t dma_addr, u8 index) 512{ 513#define SNOOP_ON (0 << 0) 514#define SNOOP_OFF (1 << 0) 515/* 516 * Sbox Smpt Reg Bits: 517 * Bits 31:2 Host address 518 * Bits 1 RSVD 519 * Bits 0 No snoop 520 */ 521#define BUILD_SMPT(NO_SNOOP, HOST_ADDR) \ 522 (u32)(((HOST_ADDR) << 2) | ((NO_SNOOP) & 0x01)) 523 524 uint32_t smpt_reg_val = BUILD_SMPT(SNOOP_ON, 525 dma_addr >> mdev->smpt->info.page_shift); 526 mic_mmio_write(&mdev->mmio, smpt_reg_val, 527 MIC_X100_SBOX_BASE_ADDRESS + 528 MIC_X100_SBOX_SMPT00 + (4 * index)); 529} 530 531/** 532 * mic_x100_smpt_hw_init - Initialize SMPT X100 specific fields. 533 * @mdev: pointer to mic_device instance 534 * 535 * RETURNS: none. 536 */ 537static void mic_x100_smpt_hw_init(struct mic_device *mdev) 538{ 539 struct mic_smpt_hw_info *info = &mdev->smpt->info; 540 541 info->num_reg = 32; 542 info->page_shift = 34; 543 info->page_size = (1ULL << info->page_shift); 544 info->base = 0x8000000000ULL; 545} 546 547struct mic_smpt_ops mic_x100_smpt_ops = { 548 .init = mic_x100_smpt_hw_init, 549 .set = mic_x100_smpt_set, 550}; 551 552static bool mic_x100_dma_filter(struct dma_chan *chan, void *param) 553{ 554 if (chan->device->dev->parent == (struct device *)param) 555 return true; 556 return false; 557} 558 559struct mic_hw_ops mic_x100_ops = { 560 .aper_bar = MIC_X100_APER_BAR, 561 .mmio_bar = MIC_X100_MMIO_BAR, 562 .read_spad = mic_x100_read_spad, 563 .write_spad = mic_x100_write_spad, 564 .send_intr = mic_x100_send_intr, 565 .ack_interrupt = mic_x100_ack_interrupt, 566 .intr_workarounds = mic_x100_intr_workarounds, 567 .reset = mic_x100_hw_reset, 568 .reset_fw_ready = mic_x100_reset_fw_ready, 569 .is_fw_ready = mic_x100_is_fw_ready, 570 .send_firmware_intr = mic_x100_send_firmware_intr, 571 .load_mic_fw = mic_x100_load_firmware, 572 .get_postcode = mic_x100_get_postcode, 573 .dma_filter = mic_x100_dma_filter, 574}; 575 576struct mic_hw_intr_ops mic_x100_intr_ops = { 577 .intr_init = mic_x100_hw_intr_init, 578 .enable_interrupts = mic_x100_enable_interrupts, 579 .disable_interrupts = mic_x100_disable_interrupts, 580 .program_msi_to_src_map = mic_x100_program_msi_to_src_map, 581 .read_msi_to_src_map = mic_x100_read_msi_to_src_map, 582}; 583