1/* 2 * Intel MIC Platform Software Stack (MPSS) 3 * 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, version 2, as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * General Public License for more details. 14 * 15 * The full GNU General Public License is included in this distribution in 16 * the file called "COPYING". 17 * 18 * Disclaimer: The codes contained in these modules may be specific to 19 * the Intel Software Development Platform codenamed: Knights Ferry, and 20 * the Intel product codenamed: Knights Corner, and are not backward 21 * compatible with other Intel products. Additionally, Intel will NOT 22 * support the codes or instruction set in future products. 23 * 24 * Adapted from: 25 * 26 * virtio for kvm on s390 27 * 28 * Copyright IBM Corp. 2008 29 * 30 * This program is free software; you can redistribute it and/or modify 31 * it under the terms of the GNU General Public License (version 2 only) 32 * as published by the Free Software Foundation. 33 * 34 * Author(s): Christian Borntraeger <borntraeger@de.ibm.com> 35 * 36 * Intel MIC Card driver. 37 * 38 */ 39#include <linux/delay.h> 40#include <linux/slab.h> 41#include <linux/virtio_config.h> 42 43#include "../common/mic_dev.h" 44#include "mic_virtio.h" 45 46#define VIRTIO_SUBCODE_64 0x0D00 47 48#define MIC_MAX_VRINGS 4 49struct mic_vdev { 50 struct virtio_device vdev; 51 struct mic_device_desc __iomem *desc; 52 struct mic_device_ctrl __iomem *dc; 53 struct mic_device *mdev; 54 void __iomem *vr[MIC_MAX_VRINGS]; 55 int used_size[MIC_MAX_VRINGS]; 56 struct completion reset_done; 57 struct mic_irq *virtio_cookie; 58 int c2h_vdev_db; 59}; 60 61static struct mic_irq *virtio_config_cookie; 62#define to_micvdev(vd) container_of(vd, struct mic_vdev, vdev) 63 64/* Helper API to obtain the parent of the virtio device */ 65static inline struct device *mic_dev(struct mic_vdev *mvdev) 66{ 67 return mvdev->vdev.dev.parent; 68} 69 70/* This gets the device's feature bits. */ 71static u64 mic_get_features(struct virtio_device *vdev) 72{ 73 unsigned int i, bits; 74 u32 features = 0; 75 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 76 u8 __iomem *in_features = mic_vq_features(desc); 77 int feature_len = ioread8(&desc->feature_len); 78 79 bits = min_t(unsigned, feature_len, sizeof(features)) * 8; 80 for (i = 0; i < bits; i++) 81 if (ioread8(&in_features[i / 8]) & (BIT(i % 8))) 82 features |= BIT(i); 83 84 return features; 85} 86 87static int mic_finalize_features(struct virtio_device *vdev) 88{ 89 unsigned int i, bits; 90 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 91 u8 feature_len = ioread8(&desc->feature_len); 92 /* Second half of bitmap is features we accept. */ 93 u8 __iomem *out_features = 94 mic_vq_features(desc) + feature_len; 95 96 /* Give virtio_ring a chance to accept features. */ 97 vring_transport_features(vdev); 98 99 /* Make sure we don't have any features > 32 bits! */ 100 BUG_ON((u32)vdev->features != vdev->features); 101 102 memset_io(out_features, 0, feature_len); 103 bits = min_t(unsigned, feature_len, 104 sizeof(vdev->features)) * 8; 105 for (i = 0; i < bits; i++) { 106 if (__virtio_test_bit(vdev, i)) 107 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)), 108 &out_features[i / 8]); 109 } 110 111 return 0; 112} 113 114/* 115 * Reading and writing elements in config space 116 */ 117static void mic_get(struct virtio_device *vdev, unsigned int offset, 118 void *buf, unsigned len) 119{ 120 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 121 122 if (offset + len > ioread8(&desc->config_len)) 123 return; 124 memcpy_fromio(buf, mic_vq_configspace(desc) + offset, len); 125} 126 127static void mic_set(struct virtio_device *vdev, unsigned int offset, 128 const void *buf, unsigned len) 129{ 130 struct mic_device_desc __iomem *desc = to_micvdev(vdev)->desc; 131 132 if (offset + len > ioread8(&desc->config_len)) 133 return; 134 memcpy_toio(mic_vq_configspace(desc) + offset, buf, len); 135} 136 137/* 138 * The operations to get and set the status word just access the status 139 * field of the device descriptor. set_status also interrupts the host 140 * to tell about status changes. 141 */ 142static u8 mic_get_status(struct virtio_device *vdev) 143{ 144 return ioread8(&to_micvdev(vdev)->desc->status); 145} 146 147static void mic_set_status(struct virtio_device *vdev, u8 status) 148{ 149 struct mic_vdev *mvdev = to_micvdev(vdev); 150 if (!status) 151 return; 152 iowrite8(status, &mvdev->desc->status); 153 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 154} 155 156/* Inform host on a virtio device reset and wait for ack from host */ 157static void mic_reset_inform_host(struct virtio_device *vdev) 158{ 159 struct mic_vdev *mvdev = to_micvdev(vdev); 160 struct mic_device_ctrl __iomem *dc = mvdev->dc; 161 int retry; 162 163 iowrite8(0, &dc->host_ack); 164 iowrite8(1, &dc->vdev_reset); 165 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 166 167 /* Wait till host completes all card accesses and acks the reset */ 168 for (retry = 100; retry--;) { 169 if (ioread8(&dc->host_ack)) 170 break; 171 msleep(100); 172 }; 173 174 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry); 175 176 /* Reset status to 0 in case we timed out */ 177 iowrite8(0, &mvdev->desc->status); 178} 179 180static void mic_reset(struct virtio_device *vdev) 181{ 182 struct mic_vdev *mvdev = to_micvdev(vdev); 183 184 dev_dbg(mic_dev(mvdev), "%s: virtio id %d\n", 185 __func__, vdev->id.device); 186 187 mic_reset_inform_host(vdev); 188 complete_all(&mvdev->reset_done); 189} 190 191/* 192 * The virtio_ring code calls this API when it wants to notify the Host. 193 */ 194static bool mic_notify(struct virtqueue *vq) 195{ 196 struct mic_vdev *mvdev = vq->priv; 197 198 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 199 return true; 200} 201 202static void mic_del_vq(struct virtqueue *vq, int n) 203{ 204 struct mic_vdev *mvdev = to_micvdev(vq->vdev); 205 struct vring *vr = (struct vring *)(vq + 1); 206 207 free_pages((unsigned long) vr->used, get_order(mvdev->used_size[n])); 208 vring_del_virtqueue(vq); 209 mic_card_unmap(mvdev->mdev, mvdev->vr[n]); 210 mvdev->vr[n] = NULL; 211} 212 213static void mic_del_vqs(struct virtio_device *vdev) 214{ 215 struct mic_vdev *mvdev = to_micvdev(vdev); 216 struct virtqueue *vq, *n; 217 int idx = 0; 218 219 dev_dbg(mic_dev(mvdev), "%s\n", __func__); 220 221 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 222 mic_del_vq(vq, idx++); 223} 224 225/* 226 * This routine will assign vring's allocated in host/io memory. Code in 227 * virtio_ring.c however continues to access this io memory as if it were local 228 * memory without io accessors. 229 */ 230static struct virtqueue *mic_find_vq(struct virtio_device *vdev, 231 unsigned index, 232 void (*callback)(struct virtqueue *vq), 233 const char *name) 234{ 235 struct mic_vdev *mvdev = to_micvdev(vdev); 236 struct mic_vqconfig __iomem *vqconfig; 237 struct mic_vqconfig config; 238 struct virtqueue *vq; 239 void __iomem *va; 240 struct _mic_vring_info __iomem *info; 241 void *used; 242 int vr_size, _vr_size, err, magic; 243 struct vring *vr; 244 u8 type = ioread8(&mvdev->desc->type); 245 246 if (index >= ioread8(&mvdev->desc->num_vq)) 247 return ERR_PTR(-ENOENT); 248 249 if (!name) 250 return ERR_PTR(-ENOENT); 251 252 /* First assign the vring's allocated in host memory */ 253 vqconfig = mic_vq_config(mvdev->desc) + index; 254 memcpy_fromio(&config, vqconfig, sizeof(config)); 255 _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN); 256 vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info)); 257 va = mic_card_map(mvdev->mdev, le64_to_cpu(config.address), vr_size); 258 if (!va) 259 return ERR_PTR(-ENOMEM); 260 mvdev->vr[index] = va; 261 memset_io(va, 0x0, _vr_size); 262 vq = vring_new_virtqueue(index, le16_to_cpu(config.num), 263 MIC_VIRTIO_RING_ALIGN, vdev, false, 264 (void __force *)va, mic_notify, callback, 265 name); 266 if (!vq) { 267 err = -ENOMEM; 268 goto unmap; 269 } 270 info = va + _vr_size; 271 magic = ioread32(&info->magic); 272 273 if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) { 274 err = -EIO; 275 goto unmap; 276 } 277 278 /* Allocate and reassign used ring now */ 279 mvdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 + 280 sizeof(struct vring_used_elem) * 281 le16_to_cpu(config.num)); 282 used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 283 get_order(mvdev->used_size[index])); 284 if (!used) { 285 err = -ENOMEM; 286 dev_err(mic_dev(mvdev), "%s %d err %d\n", 287 __func__, __LINE__, err); 288 goto del_vq; 289 } 290 iowrite64(virt_to_phys(used), &vqconfig->used_address); 291 292 /* 293 * To reassign the used ring here we are directly accessing 294 * struct vring_virtqueue which is a private data structure 295 * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in 296 * vring_new_virtqueue() would ensure that 297 * (&vq->vring == (struct vring *) (&vq->vq + 1)); 298 */ 299 vr = (struct vring *)(vq + 1); 300 vr->used = used; 301 302 vq->priv = mvdev; 303 return vq; 304del_vq: 305 vring_del_virtqueue(vq); 306unmap: 307 mic_card_unmap(mvdev->mdev, mvdev->vr[index]); 308 return ERR_PTR(err); 309} 310 311static int mic_find_vqs(struct virtio_device *vdev, unsigned nvqs, 312 struct virtqueue *vqs[], 313 vq_callback_t *callbacks[], 314 const char *names[]) 315{ 316 struct mic_vdev *mvdev = to_micvdev(vdev); 317 struct mic_device_ctrl __iomem *dc = mvdev->dc; 318 int i, err, retry; 319 320 /* We must have this many virtqueues. */ 321 if (nvqs > ioread8(&mvdev->desc->num_vq)) 322 return -ENOENT; 323 324 for (i = 0; i < nvqs; ++i) { 325 dev_dbg(mic_dev(mvdev), "%s: %d: %s\n", 326 __func__, i, names[i]); 327 vqs[i] = mic_find_vq(vdev, i, callbacks[i], names[i]); 328 if (IS_ERR(vqs[i])) { 329 err = PTR_ERR(vqs[i]); 330 goto error; 331 } 332 } 333 334 iowrite8(1, &dc->used_address_updated); 335 /* 336 * Send an interrupt to the host to inform it that used 337 * rings have been re-assigned. 338 */ 339 mic_send_intr(mvdev->mdev, mvdev->c2h_vdev_db); 340 for (retry = 100; retry--;) { 341 if (!ioread8(&dc->used_address_updated)) 342 break; 343 msleep(100); 344 }; 345 346 dev_dbg(mic_dev(mvdev), "%s: retry: %d\n", __func__, retry); 347 if (!retry) { 348 err = -ENODEV; 349 goto error; 350 } 351 352 return 0; 353error: 354 mic_del_vqs(vdev); 355 return err; 356} 357 358/* 359 * The config ops structure as defined by virtio config 360 */ 361static struct virtio_config_ops mic_vq_config_ops = { 362 .get_features = mic_get_features, 363 .finalize_features = mic_finalize_features, 364 .get = mic_get, 365 .set = mic_set, 366 .get_status = mic_get_status, 367 .set_status = mic_set_status, 368 .reset = mic_reset, 369 .find_vqs = mic_find_vqs, 370 .del_vqs = mic_del_vqs, 371}; 372 373static irqreturn_t 374mic_virtio_intr_handler(int irq, void *data) 375{ 376 struct mic_vdev *mvdev = data; 377 struct virtqueue *vq; 378 379 mic_ack_interrupt(mvdev->mdev); 380 list_for_each_entry(vq, &mvdev->vdev.vqs, list) 381 vring_interrupt(0, vq); 382 383 return IRQ_HANDLED; 384} 385 386static void mic_virtio_release_dev(struct device *_d) 387{ 388 /* 389 * No need for a release method similar to virtio PCI. 390 * Provide an empty one to avoid getting a warning from core. 391 */ 392} 393 394/* 395 * adds a new device and register it with virtio 396 * appropriate drivers are loaded by the device model 397 */ 398static int mic_add_device(struct mic_device_desc __iomem *d, 399 unsigned int offset, struct mic_driver *mdrv) 400{ 401 struct mic_vdev *mvdev; 402 int ret; 403 int virtio_db; 404 u8 type = ioread8(&d->type); 405 406 mvdev = kzalloc(sizeof(*mvdev), GFP_KERNEL); 407 if (!mvdev) { 408 dev_err(mdrv->dev, "Cannot allocate mic dev %u type %u\n", 409 offset, type); 410 return -ENOMEM; 411 } 412 413 mvdev->mdev = &mdrv->mdev; 414 mvdev->vdev.dev.parent = mdrv->dev; 415 mvdev->vdev.dev.release = mic_virtio_release_dev; 416 mvdev->vdev.id.device = type; 417 mvdev->vdev.config = &mic_vq_config_ops; 418 mvdev->desc = d; 419 mvdev->dc = (void __iomem *)d + mic_aligned_desc_size(d); 420 init_completion(&mvdev->reset_done); 421 422 virtio_db = mic_next_card_db(); 423 mvdev->virtio_cookie = mic_request_card_irq(mic_virtio_intr_handler, 424 NULL, "virtio intr", mvdev, virtio_db); 425 if (IS_ERR(mvdev->virtio_cookie)) { 426 ret = PTR_ERR(mvdev->virtio_cookie); 427 goto kfree; 428 } 429 iowrite8((u8)virtio_db, &mvdev->dc->h2c_vdev_db); 430 mvdev->c2h_vdev_db = ioread8(&mvdev->dc->c2h_vdev_db); 431 432 ret = register_virtio_device(&mvdev->vdev); 433 if (ret) { 434 dev_err(mic_dev(mvdev), 435 "Failed to register mic device %u type %u\n", 436 offset, type); 437 goto free_irq; 438 } 439 iowrite64((u64)mvdev, &mvdev->dc->vdev); 440 dev_dbg(mic_dev(mvdev), "%s: registered mic device %u type %u mvdev %p\n", 441 __func__, offset, type, mvdev); 442 443 return 0; 444 445free_irq: 446 mic_free_card_irq(mvdev->virtio_cookie, mvdev); 447kfree: 448 kfree(mvdev); 449 return ret; 450} 451 452/* 453 * match for a mic device with a specific desc pointer 454 */ 455static int mic_match_desc(struct device *dev, void *data) 456{ 457 struct virtio_device *vdev = dev_to_virtio(dev); 458 struct mic_vdev *mvdev = to_micvdev(vdev); 459 460 return mvdev->desc == (void __iomem *)data; 461} 462 463static void mic_handle_config_change(struct mic_device_desc __iomem *d, 464 unsigned int offset, struct mic_driver *mdrv) 465{ 466 struct mic_device_ctrl __iomem *dc 467 = (void __iomem *)d + mic_aligned_desc_size(d); 468 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev); 469 470 if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED) 471 return; 472 473 dev_dbg(mdrv->dev, "%s %d\n", __func__, __LINE__); 474 virtio_config_changed(&mvdev->vdev); 475 iowrite8(1, &dc->guest_ack); 476} 477 478/* 479 * removes a virtio device if a hot remove event has been 480 * requested by the host. 481 */ 482static int mic_remove_device(struct mic_device_desc __iomem *d, 483 unsigned int offset, struct mic_driver *mdrv) 484{ 485 struct mic_device_ctrl __iomem *dc 486 = (void __iomem *)d + mic_aligned_desc_size(d); 487 struct mic_vdev *mvdev = (struct mic_vdev *)ioread64(&dc->vdev); 488 u8 status; 489 int ret = -1; 490 491 if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) { 492 dev_dbg(mdrv->dev, 493 "%s %d config_change %d type %d mvdev %p\n", 494 __func__, __LINE__, 495 ioread8(&dc->config_change), ioread8(&d->type), mvdev); 496 497 status = ioread8(&d->status); 498 reinit_completion(&mvdev->reset_done); 499 unregister_virtio_device(&mvdev->vdev); 500 mic_free_card_irq(mvdev->virtio_cookie, mvdev); 501 if (status & VIRTIO_CONFIG_S_DRIVER_OK) 502 wait_for_completion(&mvdev->reset_done); 503 kfree(mvdev); 504 iowrite8(1, &dc->guest_ack); 505 dev_dbg(mdrv->dev, "%s %d guest_ack %d\n", 506 __func__, __LINE__, ioread8(&dc->guest_ack)); 507 ret = 0; 508 } 509 510 return ret; 511} 512 513#define REMOVE_DEVICES true 514 515static void mic_scan_devices(struct mic_driver *mdrv, bool remove) 516{ 517 s8 type; 518 unsigned int i; 519 struct mic_device_desc __iomem *d; 520 struct mic_device_ctrl __iomem *dc; 521 struct device *dev; 522 int ret; 523 524 for (i = sizeof(struct mic_bootparam); i < MIC_DP_SIZE; 525 i += mic_total_desc_size(d)) { 526 d = mdrv->dp + i; 527 dc = (void __iomem *)d + mic_aligned_desc_size(d); 528 /* 529 * This read barrier is paired with the corresponding write 530 * barrier on the host which is inserted before adding or 531 * removing a virtio device descriptor, by updating the type. 532 */ 533 rmb(); 534 type = ioread8(&d->type); 535 536 /* end of list */ 537 if (type == 0) 538 break; 539 540 if (type == -1) 541 continue; 542 543 /* device already exists */ 544 dev = device_find_child(mdrv->dev, (void __force *)d, 545 mic_match_desc); 546 if (dev) { 547 if (remove) 548 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE, 549 &dc->config_change); 550 put_device(dev); 551 mic_handle_config_change(d, i, mdrv); 552 ret = mic_remove_device(d, i, mdrv); 553 if (!ret && !remove) 554 iowrite8(-1, &d->type); 555 if (remove) { 556 iowrite8(0, &dc->config_change); 557 iowrite8(0, &dc->guest_ack); 558 } 559 continue; 560 } 561 562 /* new device */ 563 dev_dbg(mdrv->dev, "%s %d Adding new virtio device %p\n", 564 __func__, __LINE__, d); 565 if (!remove) 566 mic_add_device(d, i, mdrv); 567 } 568} 569 570/* 571 * mic_hotplug_device tries to find changes in the device page. 572 */ 573static void mic_hotplug_devices(struct work_struct *work) 574{ 575 struct mic_driver *mdrv = container_of(work, 576 struct mic_driver, hotplug_work); 577 578 mic_scan_devices(mdrv, !REMOVE_DEVICES); 579} 580 581/* 582 * Interrupt handler for hot plug/config changes etc. 583 */ 584static irqreturn_t 585mic_extint_handler(int irq, void *data) 586{ 587 struct mic_driver *mdrv = (struct mic_driver *)data; 588 589 dev_dbg(mdrv->dev, "%s %d hotplug work\n", 590 __func__, __LINE__); 591 mic_ack_interrupt(&mdrv->mdev); 592 schedule_work(&mdrv->hotplug_work); 593 return IRQ_HANDLED; 594} 595 596/* 597 * Init function for virtio 598 */ 599int mic_devices_init(struct mic_driver *mdrv) 600{ 601 int rc; 602 struct mic_bootparam __iomem *bootparam; 603 int config_db; 604 605 INIT_WORK(&mdrv->hotplug_work, mic_hotplug_devices); 606 mic_scan_devices(mdrv, !REMOVE_DEVICES); 607 608 config_db = mic_next_card_db(); 609 virtio_config_cookie = mic_request_card_irq(mic_extint_handler, NULL, 610 "virtio_config_intr", mdrv, 611 config_db); 612 if (IS_ERR(virtio_config_cookie)) { 613 rc = PTR_ERR(virtio_config_cookie); 614 goto exit; 615 } 616 617 bootparam = mdrv->dp; 618 iowrite8(config_db, &bootparam->h2c_config_db); 619 return 0; 620exit: 621 return rc; 622} 623 624/* 625 * Uninit function for virtio 626 */ 627void mic_devices_uninit(struct mic_driver *mdrv) 628{ 629 struct mic_bootparam __iomem *bootparam = mdrv->dp; 630 iowrite8(-1, &bootparam->h2c_config_db); 631 mic_free_card_irq(virtio_config_cookie, mdrv); 632 flush_work(&mdrv->hotplug_work); 633 mic_scan_devices(mdrv, REMOVE_DEVICES); 634} 635