drivers/ntb/hw/amd/ntb_hw

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
ndev_mw_to_bar
amd_ntb_mw_count
amd_ntb_mw_get_align
amd_ntb_mw_set_trans
amd_link_is_up
amd_ntb_link_is_up
amd_ntb_link_enable
amd_ntb_link_disable
amd_ntb_peer_mw_count
amd_ntb_peer_mw_get_addr
amd_ntb_db_valid_mask
amd_ntb_db_vector_count
amd_ntb_db_vector_mask
amd_ntb_db_read
amd_ntb_db_clear
amd_ntb_db_set_mask
amd_ntb_db_clear_mask
amd_ntb_peer_db_set
amd_ntb_spad_count
amd_ntb_spad_read
amd_ntb_spad_write
amd_ntb_peer_spad_read
amd_ntb_peer_spad_write
amd_ack_smu
amd_handle_event
ndev_interrupt
ndev_vec_isr
ndev_irq_isr
ndev_init_isr
ndev_deinit_isr
ndev_debugfs_read
ndev_init_debugfs
ndev_deinit_debugfs
ndev_init_struct
amd_poll_link
amd_link_hb
amd_init_isr
amd_init_side_info
amd_deinit_side_info
amd_init_ntb
amd_get_topo
amd_init_dev
amd_deinit_dev
amd_ntb_init_pci
amd_ntb_deinit_pci
amd_ntb_pci_probe
amd_ntb_pci_remove
amd_ntb_pci_driver_init
amd_ntb_pci_driver_exit
   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
   8  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
   9  *
  10  *   This program is free software; you can redistribute it and/or modify
  11  *   it under the terms of version 2 of the GNU General Public License as
  12  *   published by the Free Software Foundation.
  13  *
  14  *   BSD LICENSE
  15  *
  16  *   Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
  17  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  18  *
  19  *   Redistribution and use in source and binary forms, with or without
  20  *   modification, are permitted provided that the following conditions
  21  *   are met:
  22  *
  23  *     * Redistributions of source code must retain the above copyright
  24  *       notice, this list of conditions and the following disclaimer.
  25  *     * Redistributions in binary form must reproduce the above copy
  26  *       notice, this list of conditions and the following disclaimer in
  27  *       the documentation and/or other materials provided with the
  28  *       distribution.
  29  *     * Neither the name of AMD Corporation nor the names of its
  30  *       contributors may be used to endorse or promote products derived
  31  *       from this software without specific prior written permission.
  32  *
  33  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44  *
  45  * AMD PCIe NTB Linux driver
  46  *
  47  * Contact Information:
  48  * Xiangliang Yu <Xiangliang.Yu@amd.com>
  49  */
  50 
  51 #include <linux/debugfs.h>
  52 #include <linux/delay.h>
  53 #include <linux/init.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/module.h>
  56 #include <linux/acpi.h>
  57 #include <linux/pci.h>
  58 #include <linux/random.h>
  59 #include <linux/slab.h>
  60 #include <linux/ntb.h>
  61 
  62 #include "ntb_hw_amd.h"
  63 
  64 #define NTB_NAME        "ntb_hw_amd"
  65 #define NTB_DESC        "AMD(R) PCI-E Non-Transparent Bridge Driver"
  66 #define NTB_VER         "1.0"
  67 
  68 MODULE_DESCRIPTION(NTB_DESC);
  69 MODULE_VERSION(NTB_VER);
  70 MODULE_LICENSE("Dual BSD/GPL");
  71 MODULE_AUTHOR("AMD Inc.");
  72 
  73 static const struct file_operations amd_ntb_debugfs_info;
  74 static struct dentry *debugfs_dir;
  75 
  76 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
  77 {
  78         if (idx < 0 || idx > ndev->mw_count)
  79                 return -EINVAL;
  80 
  81         return ndev->dev_data->mw_idx << idx;
  82 }
  83 
  84 static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
  85 {
  86         if (pidx != NTB_DEF_PEER_IDX)
  87                 return -EINVAL;
  88 
  89         return ntb_ndev(ntb)->mw_count;
  90 }
  91 
  92 static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
  93                                 resource_size_t *addr_align,
  94                                 resource_size_t *size_align,
  95                                 resource_size_t *size_max)
  96 {
  97         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
  98         int bar;
  99 
 100         if (pidx != NTB_DEF_PEER_IDX)
 101                 return -EINVAL;
 102 
 103         bar = ndev_mw_to_bar(ndev, idx);
 104         if (bar < 0)
 105                 return bar;
 106 
 107         if (addr_align)
 108                 *addr_align = SZ_4K;
 109 
 110         if (size_align)
 111                 *size_align = 1;
 112 
 113         if (size_max)
 114                 *size_max = pci_resource_len(ndev->ntb.pdev, bar);
 115 
 116         return 0;
 117 }
 118 
 119 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
 120                                 dma_addr_t addr, resource_size_t size)
 121 {
 122         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 123         unsigned long xlat_reg, limit_reg = 0;
 124         resource_size_t mw_size;
 125         void __iomem *mmio, *peer_mmio;
 126         u64 base_addr, limit, reg_val;
 127         int bar;
 128 
 129         if (pidx != NTB_DEF_PEER_IDX)
 130                 return -EINVAL;
 131 
 132         bar = ndev_mw_to_bar(ndev, idx);
 133         if (bar < 0)
 134                 return bar;
 135 
 136         mw_size = pci_resource_len(ntb->pdev, bar);
 137 
 138         /* make sure the range fits in the usable mw size */
 139         if (size > mw_size)
 140                 return -EINVAL;
 141 
 142         mmio = ndev->self_mmio;
 143         peer_mmio = ndev->peer_mmio;
 144 
 145         base_addr = pci_resource_start(ntb->pdev, bar);
 146 
 147         if (bar != 1) {
 148                 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
 149                 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
 150 
 151                 /* Set the limit if supported */
 152                 limit = size;
 153 
 154                 /* set and verify setting the translation address */
 155                 write64(addr, peer_mmio + xlat_reg);
 156                 reg_val = read64(peer_mmio + xlat_reg);
 157                 if (reg_val != addr) {
 158                         write64(0, peer_mmio + xlat_reg);
 159                         return -EIO;
 160                 }
 161 
 162                 /* set and verify setting the limit */
 163                 write64(limit, peer_mmio + limit_reg);
 164                 reg_val = read64(peer_mmio + limit_reg);
 165                 if (reg_val != limit) {
 166                         write64(base_addr, mmio + limit_reg);
 167                         write64(0, peer_mmio + xlat_reg);
 168                         return -EIO;
 169                 }
 170         } else {
 171                 xlat_reg = AMD_BAR1XLAT_OFFSET;
 172                 limit_reg = AMD_BAR1LMT_OFFSET;
 173 
 174                 /* Set the limit if supported */
 175                 limit = size;
 176 
 177                 /* set and verify setting the translation address */
 178                 write64(addr, peer_mmio + xlat_reg);
 179                 reg_val = read64(peer_mmio + xlat_reg);
 180                 if (reg_val != addr) {
 181                         write64(0, peer_mmio + xlat_reg);
 182                         return -EIO;
 183                 }
 184 
 185                 /* set and verify setting the limit */
 186                 writel(limit, peer_mmio + limit_reg);
 187                 reg_val = readl(peer_mmio + limit_reg);
 188                 if (reg_val != limit) {
 189                         writel(base_addr, mmio + limit_reg);
 190                         writel(0, peer_mmio + xlat_reg);
 191                         return -EIO;
 192                 }
 193         }
 194 
 195         return 0;
 196 }
 197 
 198 static int amd_link_is_up(struct amd_ntb_dev *ndev)
 199 {
 200         if (!ndev->peer_sta)
 201                 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
 202 
 203         if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
 204                 ndev->peer_sta = 0;
 205                 return 1;
 206         }
 207 
 208         /* If peer_sta is reset or D0 event, the ISR has
 209          * started a timer to check link status of hardware.
 210          * So here just clear status bit. And if peer_sta is
 211          * D3 or PME_TO, D0/reset event will be happened when
 212          * system wakeup/poweron, so do nothing here.
 213          */
 214         if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
 215                 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
 216         else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))
 217                 ndev->peer_sta = 0;
 218 
 219         return 0;
 220 }
 221 
 222 static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
 223                               enum ntb_speed *speed,
 224                               enum ntb_width *width)
 225 {
 226         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 227         int ret = 0;
 228 
 229         if (amd_link_is_up(ndev)) {
 230                 if (speed)
 231                         *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
 232                 if (width)
 233                         *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
 234 
 235                 dev_dbg(&ntb->pdev->dev, "link is up.\n");
 236 
 237                 ret = 1;
 238         } else {
 239                 if (speed)
 240                         *speed = NTB_SPEED_NONE;
 241                 if (width)
 242                         *width = NTB_WIDTH_NONE;
 243 
 244                 dev_dbg(&ntb->pdev->dev, "link is down.\n");
 245         }
 246 
 247         return ret;
 248 }
 249 
 250 static int amd_ntb_link_enable(struct ntb_dev *ntb,
 251                                enum ntb_speed max_speed,
 252                                enum ntb_width max_width)
 253 {
 254         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 255         void __iomem *mmio = ndev->self_mmio;
 256         u32 ntb_ctl;
 257 
 258         /* Enable event interrupt */
 259         ndev->int_mask &= ~AMD_EVENT_INTMASK;
 260         writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 261 
 262         if (ndev->ntb.topo == NTB_TOPO_SEC)
 263                 return -EINVAL;
 264         dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
 265 
 266         ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
 267         ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
 268         writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
 269 
 270         return 0;
 271 }
 272 
 273 static int amd_ntb_link_disable(struct ntb_dev *ntb)
 274 {
 275         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 276         void __iomem *mmio = ndev->self_mmio;
 277         u32 ntb_ctl;
 278 
 279         /* Disable event interrupt */
 280         ndev->int_mask |= AMD_EVENT_INTMASK;
 281         writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 282 
 283         if (ndev->ntb.topo == NTB_TOPO_SEC)
 284                 return -EINVAL;
 285         dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
 286 
 287         ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
 288         ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
 289         writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
 290 
 291         return 0;
 292 }
 293 
 294 static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
 295 {
 296         /* The same as for inbound MWs */
 297         return ntb_ndev(ntb)->mw_count;
 298 }
 299 
 300 static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
 301                                     phys_addr_t *base, resource_size_t *size)
 302 {
 303         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 304         int bar;
 305 
 306         bar = ndev_mw_to_bar(ndev, idx);
 307         if (bar < 0)
 308                 return bar;
 309 
 310         if (base)
 311                 *base = pci_resource_start(ndev->ntb.pdev, bar);
 312 
 313         if (size)
 314                 *size = pci_resource_len(ndev->ntb.pdev, bar);
 315 
 316         return 0;
 317 }
 318 
 319 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
 320 {
 321         return ntb_ndev(ntb)->db_valid_mask;
 322 }
 323 
 324 static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
 325 {
 326         return ntb_ndev(ntb)->db_count;
 327 }
 328 
 329 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
 330 {
 331         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 332 
 333         if (db_vector < 0 || db_vector > ndev->db_count)
 334                 return 0;
 335 
 336         return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector);
 337 }
 338 
 339 static u64 amd_ntb_db_read(struct ntb_dev *ntb)
 340 {
 341         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 342         void __iomem *mmio = ndev->self_mmio;
 343 
 344         return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
 345 }
 346 
 347 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
 348 {
 349         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 350         void __iomem *mmio = ndev->self_mmio;
 351 
 352         writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
 353 
 354         return 0;
 355 }
 356 
 357 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
 358 {
 359         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 360         void __iomem *mmio = ndev->self_mmio;
 361         unsigned long flags;
 362 
 363         if (db_bits & ~ndev->db_valid_mask)
 364                 return -EINVAL;
 365 
 366         spin_lock_irqsave(&ndev->db_mask_lock, flags);
 367         ndev->db_mask |= db_bits;
 368         writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
 369         spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
 370 
 371         return 0;
 372 }
 373 
 374 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
 375 {
 376         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 377         void __iomem *mmio = ndev->self_mmio;
 378         unsigned long flags;
 379 
 380         if (db_bits & ~ndev->db_valid_mask)
 381                 return -EINVAL;
 382 
 383         spin_lock_irqsave(&ndev->db_mask_lock, flags);
 384         ndev->db_mask &= ~db_bits;
 385         writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
 386         spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
 387 
 388         return 0;
 389 }
 390 
 391 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 392 {
 393         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 394         void __iomem *mmio = ndev->self_mmio;
 395 
 396         writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
 397 
 398         return 0;
 399 }
 400 
 401 static int amd_ntb_spad_count(struct ntb_dev *ntb)
 402 {
 403         return ntb_ndev(ntb)->spad_count;
 404 }
 405 
 406 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
 407 {
 408         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 409         void __iomem *mmio = ndev->self_mmio;
 410         u32 offset;
 411 
 412         if (idx < 0 || idx >= ndev->spad_count)
 413                 return 0;
 414 
 415         offset = ndev->self_spad + (idx << 2);
 416         return readl(mmio + AMD_SPAD_OFFSET + offset);
 417 }
 418 
 419 static int amd_ntb_spad_write(struct ntb_dev *ntb,
 420                               int idx, u32 val)
 421 {
 422         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 423         void __iomem *mmio = ndev->self_mmio;
 424         u32 offset;
 425 
 426         if (idx < 0 || idx >= ndev->spad_count)
 427                 return -EINVAL;
 428 
 429         offset = ndev->self_spad + (idx << 2);
 430         writel(val, mmio + AMD_SPAD_OFFSET + offset);
 431 
 432         return 0;
 433 }
 434 
 435 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
 436 {
 437         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 438         void __iomem *mmio = ndev->self_mmio;
 439         u32 offset;
 440 
 441         if (sidx < 0 || sidx >= ndev->spad_count)
 442                 return -EINVAL;
 443 
 444         offset = ndev->peer_spad + (sidx << 2);
 445         return readl(mmio + AMD_SPAD_OFFSET + offset);
 446 }
 447 
 448 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
 449                                    int sidx, u32 val)
 450 {
 451         struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 452         void __iomem *mmio = ndev->self_mmio;
 453         u32 offset;
 454 
 455         if (sidx < 0 || sidx >= ndev->spad_count)
 456                 return -EINVAL;
 457 
 458         offset = ndev->peer_spad + (sidx << 2);
 459         writel(val, mmio + AMD_SPAD_OFFSET + offset);
 460 
 461         return 0;
 462 }
 463 
 464 static const struct ntb_dev_ops amd_ntb_ops = {
 465         .mw_count               = amd_ntb_mw_count,
 466         .mw_get_align           = amd_ntb_mw_get_align,
 467         .mw_set_trans           = amd_ntb_mw_set_trans,
 468         .peer_mw_count          = amd_ntb_peer_mw_count,
 469         .peer_mw_get_addr       = amd_ntb_peer_mw_get_addr,
 470         .link_is_up             = amd_ntb_link_is_up,
 471         .link_enable            = amd_ntb_link_enable,
 472         .link_disable           = amd_ntb_link_disable,
 473         .db_valid_mask          = amd_ntb_db_valid_mask,
 474         .db_vector_count        = amd_ntb_db_vector_count,
 475         .db_vector_mask         = amd_ntb_db_vector_mask,
 476         .db_read                = amd_ntb_db_read,
 477         .db_clear               = amd_ntb_db_clear,
 478         .db_set_mask            = amd_ntb_db_set_mask,
 479         .db_clear_mask          = amd_ntb_db_clear_mask,
 480         .peer_db_set            = amd_ntb_peer_db_set,
 481         .spad_count             = amd_ntb_spad_count,
 482         .spad_read              = amd_ntb_spad_read,
 483         .spad_write             = amd_ntb_spad_write,
 484         .peer_spad_read         = amd_ntb_peer_spad_read,
 485         .peer_spad_write        = amd_ntb_peer_spad_write,
 486 };
 487 
 488 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
 489 {
 490         void __iomem *mmio = ndev->self_mmio;
 491         int reg;
 492 
 493         reg = readl(mmio + AMD_SMUACK_OFFSET);
 494         reg |= bit;
 495         writel(reg, mmio + AMD_SMUACK_OFFSET);
 496 
 497         ndev->peer_sta |= bit;
 498 }
 499 
 500 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
 501 {
 502         void __iomem *mmio = ndev->self_mmio;
 503         struct device *dev = &ndev->ntb.pdev->dev;
 504         u32 status;
 505 
 506         status = readl(mmio + AMD_INTSTAT_OFFSET);
 507         if (!(status & AMD_EVENT_INTMASK))
 508                 return;
 509 
 510         dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
 511 
 512         status &= AMD_EVENT_INTMASK;
 513         switch (status) {
 514         case AMD_PEER_FLUSH_EVENT:
 515                 dev_info(dev, "Flush is done.\n");
 516                 break;
 517         case AMD_PEER_RESET_EVENT:
 518                 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
 519 
 520                 /* link down first */
 521                 ntb_link_event(&ndev->ntb);
 522                 /* polling peer status */
 523                 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
 524 
 525                 break;
 526         case AMD_PEER_D3_EVENT:
 527         case AMD_PEER_PMETO_EVENT:
 528         case AMD_LINK_UP_EVENT:
 529         case AMD_LINK_DOWN_EVENT:
 530                 amd_ack_smu(ndev, status);
 531 
 532                 /* link down */
 533                 ntb_link_event(&ndev->ntb);
 534 
 535                 break;
 536         case AMD_PEER_D0_EVENT:
 537                 mmio = ndev->peer_mmio;
 538                 status = readl(mmio + AMD_PMESTAT_OFFSET);
 539                 /* check if this is WAKEUP event */
 540                 if (status & 0x1)
 541                         dev_info(dev, "Wakeup is done.\n");
 542 
 543                 amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
 544 
 545                 /* start a timer to poll link status */
 546                 schedule_delayed_work(&ndev->hb_timer,
 547                                       AMD_LINK_HB_TIMEOUT);
 548                 break;
 549         default:
 550                 dev_info(dev, "event status = 0x%x.\n", status);
 551                 break;
 552         }
 553 }
 554 
 555 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
 556 {
 557         dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
 558 
 559         if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
 560                 amd_handle_event(ndev, vec);
 561 
 562         if (vec < AMD_DB_CNT)
 563                 ntb_db_event(&ndev->ntb, vec);
 564 
 565         return IRQ_HANDLED;
 566 }
 567 
 568 static irqreturn_t ndev_vec_isr(int irq, void *dev)
 569 {
 570         struct amd_ntb_vec *nvec = dev;
 571 
 572         return ndev_interrupt(nvec->ndev, nvec->num);
 573 }
 574 
 575 static irqreturn_t ndev_irq_isr(int irq, void *dev)
 576 {
 577         struct amd_ntb_dev *ndev = dev;
 578 
 579         return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
 580 }
 581 
 582 static int ndev_init_isr(struct amd_ntb_dev *ndev,
 583                          int msix_min, int msix_max)
 584 {
 585         struct pci_dev *pdev;
 586         int rc, i, msix_count, node;
 587 
 588         pdev = ndev->ntb.pdev;
 589 
 590         node = dev_to_node(&pdev->dev);
 591 
 592         ndev->db_mask = ndev->db_valid_mask;
 593 
 594         /* Try to set up msix irq */
 595         ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
 596                                  GFP_KERNEL, node);
 597         if (!ndev->vec)
 598                 goto err_msix_vec_alloc;
 599 
 600         ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
 601                                   GFP_KERNEL, node);
 602         if (!ndev->msix)
 603                 goto err_msix_alloc;
 604 
 605         for (i = 0; i < msix_max; ++i)
 606                 ndev->msix[i].entry = i;
 607 
 608         msix_count = pci_enable_msix_range(pdev, ndev->msix,
 609                                            msix_min, msix_max);
 610         if (msix_count < 0)
 611                 goto err_msix_enable;
 612 
 613         /* NOTE: Disable MSIX if msix count is less than 16 because of
 614          * hardware limitation.
 615          */
 616         if (msix_count < msix_min) {
 617                 pci_disable_msix(pdev);
 618                 goto err_msix_enable;
 619         }
 620 
 621         for (i = 0; i < msix_count; ++i) {
 622                 ndev->vec[i].ndev = ndev;
 623                 ndev->vec[i].num = i;
 624                 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 625                                  "ndev_vec_isr", &ndev->vec[i]);
 626                 if (rc)
 627                         goto err_msix_request;
 628         }
 629 
 630         dev_dbg(&pdev->dev, "Using msix interrupts\n");
 631         ndev->db_count = msix_min;
 632         ndev->msix_vec_count = msix_max;
 633         return 0;
 634 
 635 err_msix_request:
 636         while (i-- > 0)
 637                 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 638         pci_disable_msix(pdev);
 639 err_msix_enable:
 640         kfree(ndev->msix);
 641 err_msix_alloc:
 642         kfree(ndev->vec);
 643 err_msix_vec_alloc:
 644         ndev->msix = NULL;
 645         ndev->vec = NULL;
 646 
 647         /* Try to set up msi irq */
 648         rc = pci_enable_msi(pdev);
 649         if (rc)
 650                 goto err_msi_enable;
 651 
 652         rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 653                          "ndev_irq_isr", ndev);
 654         if (rc)
 655                 goto err_msi_request;
 656 
 657         dev_dbg(&pdev->dev, "Using msi interrupts\n");
 658         ndev->db_count = 1;
 659         ndev->msix_vec_count = 1;
 660         return 0;
 661 
 662 err_msi_request:
 663         pci_disable_msi(pdev);
 664 err_msi_enable:
 665 
 666         /* Try to set up intx irq */
 667         pci_intx(pdev, 1);
 668 
 669         rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 670                          "ndev_irq_isr", ndev);
 671         if (rc)
 672                 goto err_intx_request;
 673 
 674         dev_dbg(&pdev->dev, "Using intx interrupts\n");
 675         ndev->db_count = 1;
 676         ndev->msix_vec_count = 1;
 677         return 0;
 678 
 679 err_intx_request:
 680         return rc;
 681 }
 682 
 683 static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
 684 {
 685         struct pci_dev *pdev;
 686         void __iomem *mmio = ndev->self_mmio;
 687         int i;
 688 
 689         pdev = ndev->ntb.pdev;
 690 
 691         /* Mask all doorbell interrupts */
 692         ndev->db_mask = ndev->db_valid_mask;
 693         writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
 694 
 695         if (ndev->msix) {
 696                 i = ndev->msix_vec_count;
 697                 while (i--)
 698                         free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 699                 pci_disable_msix(pdev);
 700                 kfree(ndev->msix);
 701                 kfree(ndev->vec);
 702         } else {
 703                 free_irq(pdev->irq, ndev);
 704                 if (pci_dev_msi_enabled(pdev))
 705                         pci_disable_msi(pdev);
 706                 else
 707                         pci_intx(pdev, 0);
 708         }
 709 }
 710 
 711 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 712                                  size_t count, loff_t *offp)
 713 {
 714         struct amd_ntb_dev *ndev;
 715         void __iomem *mmio;
 716         char *buf;
 717         size_t buf_size;
 718         ssize_t ret, off;
 719         union { u64 v64; u32 v32; u16 v16; } u;
 720 
 721         ndev = filp->private_data;
 722         mmio = ndev->self_mmio;
 723 
 724         buf_size = min(count, 0x800ul);
 725 
 726         buf = kmalloc(buf_size, GFP_KERNEL);
 727         if (!buf)
 728                 return -ENOMEM;
 729 
 730         off = 0;
 731 
 732         off += scnprintf(buf + off, buf_size - off,
 733                          "NTB Device Information:\n");
 734 
 735         off += scnprintf(buf + off, buf_size - off,
 736                          "Connection Topology -\t%s\n",
 737                          ntb_topo_string(ndev->ntb.topo));
 738 
 739         off += scnprintf(buf + off, buf_size - off,
 740                          "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 741 
 742         if (!amd_link_is_up(ndev)) {
 743                 off += scnprintf(buf + off, buf_size - off,
 744                                  "Link Status -\t\tDown\n");
 745         } else {
 746                 off += scnprintf(buf + off, buf_size - off,
 747                                  "Link Status -\t\tUp\n");
 748                 off += scnprintf(buf + off, buf_size - off,
 749                                  "Link Speed -\t\tPCI-E Gen %u\n",
 750                                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
 751                 off += scnprintf(buf + off, buf_size - off,
 752                                  "Link Width -\t\tx%u\n",
 753                                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 754         }
 755 
 756         off += scnprintf(buf + off, buf_size - off,
 757                          "Memory Window Count -\t%u\n", ndev->mw_count);
 758         off += scnprintf(buf + off, buf_size - off,
 759                          "Scratchpad Count -\t%u\n", ndev->spad_count);
 760         off += scnprintf(buf + off, buf_size - off,
 761                          "Doorbell Count -\t%u\n", ndev->db_count);
 762         off += scnprintf(buf + off, buf_size - off,
 763                          "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
 764 
 765         off += scnprintf(buf + off, buf_size - off,
 766                          "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 767 
 768         u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
 769         off += scnprintf(buf + off, buf_size - off,
 770                          "Doorbell Mask -\t\t\t%#06x\n", u.v32);
 771 
 772         u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
 773         off += scnprintf(buf + off, buf_size - off,
 774                          "Doorbell Bell -\t\t\t%#06x\n", u.v32);
 775 
 776         off += scnprintf(buf + off, buf_size - off,
 777                          "\nNTB Incoming XLAT:\n");
 778 
 779         u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
 780         off += scnprintf(buf + off, buf_size - off,
 781                          "XLAT1 -\t\t%#018llx\n", u.v64);
 782 
 783         u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
 784         off += scnprintf(buf + off, buf_size - off,
 785                          "XLAT23 -\t\t%#018llx\n", u.v64);
 786 
 787         u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
 788         off += scnprintf(buf + off, buf_size - off,
 789                          "XLAT45 -\t\t%#018llx\n", u.v64);
 790 
 791         u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
 792         off += scnprintf(buf + off, buf_size - off,
 793                          "LMT1 -\t\t\t%#06x\n", u.v32);
 794 
 795         u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
 796         off += scnprintf(buf + off, buf_size - off,
 797                          "LMT23 -\t\t\t%#018llx\n", u.v64);
 798 
 799         u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
 800         off += scnprintf(buf + off, buf_size - off,
 801                          "LMT45 -\t\t\t%#018llx\n", u.v64);
 802 
 803         ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 804         kfree(buf);
 805         return ret;
 806 }
 807 
 808 static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
 809 {
 810         if (!debugfs_dir) {
 811                 ndev->debugfs_dir = NULL;
 812                 ndev->debugfs_info = NULL;
 813         } else {
 814                 ndev->debugfs_dir =
 815                         debugfs_create_dir(pci_name(ndev->ntb.pdev),
 816                                            debugfs_dir);
 817                 if (!ndev->debugfs_dir)
 818                         ndev->debugfs_info = NULL;
 819                 else
 820                         ndev->debugfs_info =
 821                                 debugfs_create_file("info", S_IRUSR,
 822                                                     ndev->debugfs_dir, ndev,
 823                                                     &amd_ntb_debugfs_info);
 824         }
 825 }
 826 
 827 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
 828 {
 829         debugfs_remove_recursive(ndev->debugfs_dir);
 830 }
 831 
 832 static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
 833                                     struct pci_dev *pdev)
 834 {
 835         ndev->ntb.pdev = pdev;
 836         ndev->ntb.topo = NTB_TOPO_NONE;
 837         ndev->ntb.ops = &amd_ntb_ops;
 838         ndev->int_mask = AMD_EVENT_INTMASK;
 839         spin_lock_init(&ndev->db_mask_lock);
 840 }
 841 
 842 static int amd_poll_link(struct amd_ntb_dev *ndev)
 843 {
 844         void __iomem *mmio = ndev->peer_mmio;
 845         u32 reg, stat;
 846         int rc;
 847 
 848         reg = readl(mmio + AMD_SIDEINFO_OFFSET);
 849         reg &= NTB_LIN_STA_ACTIVE_BIT;
 850 
 851         dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
 852 
 853         if (reg == ndev->cntl_sta)
 854                 return 0;
 855 
 856         ndev->cntl_sta = reg;
 857 
 858         rc = pci_read_config_dword(ndev->ntb.pdev,
 859                                    AMD_LINK_STATUS_OFFSET, &stat);
 860         if (rc)
 861                 return 0;
 862         ndev->lnk_sta = stat;
 863 
 864         return 1;
 865 }
 866 
 867 static void amd_link_hb(struct work_struct *work)
 868 {
 869         struct amd_ntb_dev *ndev = hb_ndev(work);
 870 
 871         if (amd_poll_link(ndev))
 872                 ntb_link_event(&ndev->ntb);
 873 
 874         if (!amd_link_is_up(ndev))
 875                 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
 876 }
 877 
 878 static int amd_init_isr(struct amd_ntb_dev *ndev)
 879 {
 880         return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
 881 }
 882 
 883 static void amd_init_side_info(struct amd_ntb_dev *ndev)
 884 {
 885         void __iomem *mmio = ndev->self_mmio;
 886         unsigned int reg;
 887 
 888         reg = readl(mmio + AMD_SIDEINFO_OFFSET);
 889         if (!(reg & AMD_SIDE_READY)) {
 890                 reg |= AMD_SIDE_READY;
 891                 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
 892         }
 893 }
 894 
 895 static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
 896 {
 897         void __iomem *mmio = ndev->self_mmio;
 898         unsigned int reg;
 899 
 900         reg = readl(mmio + AMD_SIDEINFO_OFFSET);
 901         if (reg & AMD_SIDE_READY) {
 902                 reg &= ~AMD_SIDE_READY;
 903                 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
 904                 readl(mmio + AMD_SIDEINFO_OFFSET);
 905         }
 906 }
 907 
 908 static int amd_init_ntb(struct amd_ntb_dev *ndev)
 909 {
 910         void __iomem *mmio = ndev->self_mmio;
 911 
 912         ndev->mw_count = ndev->dev_data->mw_count;
 913         ndev->spad_count = AMD_SPADS_CNT;
 914         ndev->db_count = AMD_DB_CNT;
 915 
 916         switch (ndev->ntb.topo) {
 917         case NTB_TOPO_PRI:
 918         case NTB_TOPO_SEC:
 919                 ndev->spad_count >>= 1;
 920                 if (ndev->ntb.topo == NTB_TOPO_PRI) {
 921                         ndev->self_spad = 0;
 922                         ndev->peer_spad = 0x20;
 923                 } else {
 924                         ndev->self_spad = 0x20;
 925                         ndev->peer_spad = 0;
 926                 }
 927 
 928                 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
 929                 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
 930 
 931                 break;
 932         default:
 933                 dev_err(&ndev->ntb.pdev->dev,
 934                         "AMD NTB does not support B2B mode.\n");
 935                 return -EINVAL;
 936         }
 937 
 938         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
 939 
 940         /* Mask event interrupts */
 941         writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
 942 
 943         return 0;
 944 }
 945 
 946 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
 947 {
 948         void __iomem *mmio = ndev->self_mmio;
 949         u32 info;
 950 
 951         info = readl(mmio + AMD_SIDEINFO_OFFSET);
 952         if (info & AMD_SIDE_MASK)
 953                 return NTB_TOPO_SEC;
 954         else
 955                 return NTB_TOPO_PRI;
 956 }
 957 
 958 static int amd_init_dev(struct amd_ntb_dev *ndev)
 959 {
 960         struct pci_dev *pdev;
 961         int rc = 0;
 962 
 963         pdev = ndev->ntb.pdev;
 964 
 965         ndev->ntb.topo = amd_get_topo(ndev);
 966         dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
 967                 ntb_topo_string(ndev->ntb.topo));
 968 
 969         rc = amd_init_ntb(ndev);
 970         if (rc)
 971                 return rc;
 972 
 973         rc = amd_init_isr(ndev);
 974         if (rc) {
 975                 dev_err(&pdev->dev, "fail to init isr.\n");
 976                 return rc;
 977         }
 978 
 979         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
 980 
 981         return 0;
 982 }
 983 
 984 static void amd_deinit_dev(struct amd_ntb_dev *ndev)
 985 {
 986         cancel_delayed_work_sync(&ndev->hb_timer);
 987 
 988         ndev_deinit_isr(ndev);
 989 }
 990 
 991 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
 992                             struct pci_dev *pdev)
 993 {
 994         int rc;
 995 
 996         pci_set_drvdata(pdev, ndev);
 997 
 998         rc = pci_enable_device(pdev);
 999         if (rc)
1000                 goto err_pci_enable;
1001 
1002         rc = pci_request_regions(pdev, NTB_NAME);
1003         if (rc)
1004                 goto err_pci_regions;
1005 
1006         pci_set_master(pdev);
1007 
1008         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1009         if (rc) {
1010                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1011                 if (rc)
1012                         goto err_dma_mask;
1013                 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1014         }
1015 
1016         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1017         if (rc) {
1018                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1019                 if (rc)
1020                         goto err_dma_mask;
1021                 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1022         }
1023         rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
1024                                           dma_get_mask(&pdev->dev));
1025         if (rc)
1026                 goto err_dma_mask;
1027 
1028         ndev->self_mmio = pci_iomap(pdev, 0, 0);
1029         if (!ndev->self_mmio) {
1030                 rc = -EIO;
1031                 goto err_dma_mask;
1032         }
1033         ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1034 
1035         return 0;
1036 
1037 err_dma_mask:
1038         pci_clear_master(pdev);
1039 err_pci_regions:
1040         pci_disable_device(pdev);
1041 err_pci_enable:
1042         pci_set_drvdata(pdev, NULL);
1043         return rc;
1044 }
1045 
1046 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1047 {
1048         struct pci_dev *pdev = ndev->ntb.pdev;
1049 
1050         pci_iounmap(pdev, ndev->self_mmio);
1051 
1052         pci_clear_master(pdev);
1053         pci_release_regions(pdev);
1054         pci_disable_device(pdev);
1055         pci_set_drvdata(pdev, NULL);
1056 }
1057 
1058 static int amd_ntb_pci_probe(struct pci_dev *pdev,
1059                              const struct pci_device_id *id)
1060 {
1061         struct amd_ntb_dev *ndev;
1062         int rc, node;
1063 
1064         node = dev_to_node(&pdev->dev);
1065 
1066         ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1067         if (!ndev) {
1068                 rc = -ENOMEM;
1069                 goto err_ndev;
1070         }
1071 
1072         ndev->dev_data = (struct ntb_dev_data *)id->driver_data;
1073 
1074         ndev_init_struct(ndev, pdev);
1075 
1076         rc = amd_ntb_init_pci(ndev, pdev);
1077         if (rc)
1078                 goto err_init_pci;
1079 
1080         rc = amd_init_dev(ndev);
1081         if (rc)
1082                 goto err_init_dev;
1083 
1084         /* write side info */
1085         amd_init_side_info(ndev);
1086 
1087         amd_poll_link(ndev);
1088 
1089         ndev_init_debugfs(ndev);
1090 
1091         rc = ntb_register_device(&ndev->ntb);
1092         if (rc)
1093                 goto err_register;
1094 
1095         dev_info(&pdev->dev, "NTB device registered.\n");
1096 
1097         return 0;
1098 
1099 err_register:
1100         ndev_deinit_debugfs(ndev);
1101         amd_deinit_dev(ndev);
1102 err_init_dev:
1103         amd_ntb_deinit_pci(ndev);
1104 err_init_pci:
1105         kfree(ndev);
1106 err_ndev:
1107         return rc;
1108 }
1109 
1110 static void amd_ntb_pci_remove(struct pci_dev *pdev)
1111 {
1112         struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1113 
1114         ntb_unregister_device(&ndev->ntb);
1115         ndev_deinit_debugfs(ndev);
1116         amd_deinit_side_info(ndev);
1117         amd_deinit_dev(ndev);
1118         amd_ntb_deinit_pci(ndev);
1119         kfree(ndev);
1120 }
1121 
1122 static const struct file_operations amd_ntb_debugfs_info = {
1123         .owner = THIS_MODULE,
1124         .open = simple_open,
1125         .read = ndev_debugfs_read,
1126 };
1127 
1128 static const struct ntb_dev_data dev_data[] = {
1129         { /* for device 145b */
1130                 .mw_count = 3,
1131                 .mw_idx = 1,
1132         },
1133         { /* for device 148b */
1134                 .mw_count = 2,
1135                 .mw_idx = 2,
1136         },
1137 };
1138 
1139 static const struct pci_device_id amd_ntb_pci_tbl[] = {
1140         { PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] },
1141         { PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] },
1142         { 0, }
1143 };
1144 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1145 
1146 static struct pci_driver amd_ntb_pci_driver = {
1147         .name           = KBUILD_MODNAME,
1148         .id_table       = amd_ntb_pci_tbl,
1149         .probe          = amd_ntb_pci_probe,
1150         .remove         = amd_ntb_pci_remove,
1151 };
1152 
1153 static int __init amd_ntb_pci_driver_init(void)
1154 {
1155         pr_info("%s %s\n", NTB_DESC, NTB_VER);
1156 
1157         if (debugfs_initialized())
1158                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1159 
1160         return pci_register_driver(&amd_ntb_pci_driver);
1161 }
1162 module_init(amd_ntb_pci_driver_init);
1163 
1164 static void __exit amd_ntb_pci_driver_exit(void)
1165 {
1166         pci_unregister_driver(&amd_ntb_pci_driver);
1167         debugfs_remove_recursive(debugfs_dir);
1168 }
1169 module_exit(amd_ntb_pci_driver_exit);
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/ntb/hw/amd/ntb_hw_amd.c

DEFINITIONS
