drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
parity32
dma64_dd_parity
xxd
txd
rxd
nexttxd
prevtxd
nextrxd
ntxdactive
nrxdactive
_dma_ctrlflags
_dma64_addrext
_dma_isaddrext
_dma_descriptor_align
dma_alloc_consistent
dma_align_sizetobits
dma_ringalloc
dma64_alloc
_dma_alloc
dma_attach
dma64_dd_upd
dma_detach
_dma_ddtable_init
_dma_rxenable
dma_rxinit
dma64_getnextrxp
_dma_getnextrxp
dma_rx
dma64_rxidle
dma64_txidle
dma_rxfill
dma_rxreclaim
dma_counterreset
dma_getvar
dma_txinit
dma_txsuspend
dma_txresume
dma_txsuspended
dma_txreclaim
dma_txreset
dma_rxreset
dma_txenq
ampdu_finalize
prep_ampdu_frame
dma_update_txavail
dma_txfast
dma_txflush
dma_txpending
dma_kick_tx
dma_getnexttxp
dma_walk_packets
   1 /*
   2  * Copyright (c) 2010 Broadcom Corporation
   3  *
   4  * Permission to use, copy, modify, and/or distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  11  * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  13  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  14  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16 
  17 #include <linux/slab.h>
  18 #include <linux/delay.h>
  19 #include <linux/pci.h>
  20 #include <net/cfg80211.h>
  21 #include <net/mac80211.h>
  22 
  23 #include <brcmu_utils.h>
  24 #include <aiutils.h>
  25 #include "types.h"
  26 #include "main.h"
  27 #include "dma.h"
  28 #include "soc.h"
  29 #include "scb.h"
  30 #include "ampdu.h"
  31 #include "debug.h"
  32 #include "brcms_trace_events.h"
  33 
  34 /*
  35  * dma register field offset calculation
  36  */
  37 #define DMA64REGOFFS(field)             offsetof(struct dma64regs, field)
  38 #define DMA64TXREGOFFS(di, field)       (di->d64txregbase + DMA64REGOFFS(field))
  39 #define DMA64RXREGOFFS(di, field)       (di->d64rxregbase + DMA64REGOFFS(field))
  40 
  41 /*
  42  * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within
  43  * a contiguous 8kB physical address.
  44  */
  45 #define D64RINGALIGN_BITS       13
  46 #define D64MAXRINGSZ            (1 << D64RINGALIGN_BITS)
  47 #define D64RINGALIGN            (1 << D64RINGALIGN_BITS)
  48 
  49 #define D64MAXDD        (D64MAXRINGSZ / sizeof(struct dma64desc))
  50 
  51 /* transmit channel control */
  52 #define D64_XC_XE               0x00000001      /* transmit enable */
  53 #define D64_XC_SE               0x00000002      /* transmit suspend request */
  54 #define D64_XC_LE               0x00000004      /* loopback enable */
  55 #define D64_XC_FL               0x00000010      /* flush request */
  56 #define D64_XC_PD               0x00000800      /* parity check disable */
  57 #define D64_XC_AE               0x00030000      /* address extension bits */
  58 #define D64_XC_AE_SHIFT         16
  59 
  60 /* transmit descriptor table pointer */
  61 #define D64_XP_LD_MASK          0x00000fff      /* last valid descriptor */
  62 
  63 /* transmit channel status */
  64 #define D64_XS0_CD_MASK         0x00001fff      /* current descriptor pointer */
  65 #define D64_XS0_XS_MASK         0xf0000000      /* transmit state */
  66 #define D64_XS0_XS_SHIFT                28
  67 #define D64_XS0_XS_DISABLED     0x00000000      /* disabled */
  68 #define D64_XS0_XS_ACTIVE       0x10000000      /* active */
  69 #define D64_XS0_XS_IDLE         0x20000000      /* idle wait */
  70 #define D64_XS0_XS_STOPPED      0x30000000      /* stopped */
  71 #define D64_XS0_XS_SUSP         0x40000000      /* suspend pending */
  72 
  73 #define D64_XS1_AD_MASK         0x00001fff      /* active descriptor */
  74 #define D64_XS1_XE_MASK         0xf0000000      /* transmit errors */
  75 #define D64_XS1_XE_SHIFT                28
  76 #define D64_XS1_XE_NOERR        0x00000000      /* no error */
  77 #define D64_XS1_XE_DPE          0x10000000      /* descriptor protocol error */
  78 #define D64_XS1_XE_DFU          0x20000000      /* data fifo underrun */
  79 #define D64_XS1_XE_DTE          0x30000000      /* data transfer error */
  80 #define D64_XS1_XE_DESRE        0x40000000      /* descriptor read error */
  81 #define D64_XS1_XE_COREE        0x50000000      /* core error */
  82 
  83 /* receive channel control */
  84 /* receive enable */
  85 #define D64_RC_RE               0x00000001
  86 /* receive frame offset */
  87 #define D64_RC_RO_MASK          0x000000fe
  88 #define D64_RC_RO_SHIFT         1
  89 /* direct fifo receive (pio) mode */
  90 #define D64_RC_FM               0x00000100
  91 /* separate rx header descriptor enable */
  92 #define D64_RC_SH               0x00000200
  93 /* overflow continue */
  94 #define D64_RC_OC               0x00000400
  95 /* parity check disable */
  96 #define D64_RC_PD               0x00000800
  97 /* address extension bits */
  98 #define D64_RC_AE               0x00030000
  99 #define D64_RC_AE_SHIFT         16
 100 
 101 /* flags for dma controller */
 102 /* partity enable */
 103 #define DMA_CTRL_PEN            (1 << 0)
 104 /* rx overflow continue */
 105 #define DMA_CTRL_ROC            (1 << 1)
 106 /* allow rx scatter to multiple descriptors */
 107 #define DMA_CTRL_RXMULTI        (1 << 2)
 108 /* Unframed Rx/Tx data */
 109 #define DMA_CTRL_UNFRAMED       (1 << 3)
 110 
 111 /* receive descriptor table pointer */
 112 #define D64_RP_LD_MASK          0x00000fff      /* last valid descriptor */
 113 
 114 /* receive channel status */
 115 #define D64_RS0_CD_MASK         0x00001fff      /* current descriptor pointer */
 116 #define D64_RS0_RS_MASK         0xf0000000      /* receive state */
 117 #define D64_RS0_RS_SHIFT                28
 118 #define D64_RS0_RS_DISABLED     0x00000000      /* disabled */
 119 #define D64_RS0_RS_ACTIVE       0x10000000      /* active */
 120 #define D64_RS0_RS_IDLE         0x20000000      /* idle wait */
 121 #define D64_RS0_RS_STOPPED      0x30000000      /* stopped */
 122 #define D64_RS0_RS_SUSP         0x40000000      /* suspend pending */
 123 
 124 #define D64_RS1_AD_MASK         0x0001ffff      /* active descriptor */
 125 #define D64_RS1_RE_MASK         0xf0000000      /* receive errors */
 126 #define D64_RS1_RE_SHIFT                28
 127 #define D64_RS1_RE_NOERR        0x00000000      /* no error */
 128 #define D64_RS1_RE_DPO          0x10000000      /* descriptor protocol error */
 129 #define D64_RS1_RE_DFU          0x20000000      /* data fifo overflow */
 130 #define D64_RS1_RE_DTE          0x30000000      /* data transfer error */
 131 #define D64_RS1_RE_DESRE        0x40000000      /* descriptor read error */
 132 #define D64_RS1_RE_COREE        0x50000000      /* core error */
 133 
 134 /* fifoaddr */
 135 #define D64_FA_OFF_MASK         0xffff  /* offset */
 136 #define D64_FA_SEL_MASK         0xf0000 /* select */
 137 #define D64_FA_SEL_SHIFT        16
 138 #define D64_FA_SEL_XDD          0x00000 /* transmit dma data */
 139 #define D64_FA_SEL_XDP          0x10000 /* transmit dma pointers */
 140 #define D64_FA_SEL_RDD          0x40000 /* receive dma data */
 141 #define D64_FA_SEL_RDP          0x50000 /* receive dma pointers */
 142 #define D64_FA_SEL_XFD          0x80000 /* transmit fifo data */
 143 #define D64_FA_SEL_XFP          0x90000 /* transmit fifo pointers */
 144 #define D64_FA_SEL_RFD          0xc0000 /* receive fifo data */
 145 #define D64_FA_SEL_RFP          0xd0000 /* receive fifo pointers */
 146 #define D64_FA_SEL_RSD          0xe0000 /* receive frame status data */
 147 #define D64_FA_SEL_RSP          0xf0000 /* receive frame status pointers */
 148 
 149 /* descriptor control flags 1 */
 150 #define D64_CTRL_COREFLAGS      0x0ff00000      /* core specific flags */
 151 #define D64_CTRL1_EOT           ((u32)1 << 28)  /* end of descriptor table */
 152 #define D64_CTRL1_IOC           ((u32)1 << 29)  /* interrupt on completion */
 153 #define D64_CTRL1_EOF           ((u32)1 << 30)  /* end of frame */
 154 #define D64_CTRL1_SOF           ((u32)1 << 31)  /* start of frame */
 155 
 156 /* descriptor control flags 2 */
 157 /* buffer byte count. real data len must <= 16KB */
 158 #define D64_CTRL2_BC_MASK       0x00007fff
 159 /* address extension bits */
 160 #define D64_CTRL2_AE            0x00030000
 161 #define D64_CTRL2_AE_SHIFT      16
 162 /* parity bit */
 163 #define D64_CTRL2_PARITY        0x00040000
 164 
 165 /* control flags in the range [27:20] are core-specific and not defined here */
 166 #define D64_CTRL_CORE_MASK      0x0ff00000
 167 
 168 #define D64_RX_FRM_STS_LEN      0x0000ffff      /* frame length mask */
 169 #define D64_RX_FRM_STS_OVFL     0x00800000      /* RxOverFlow */
 170 #define D64_RX_FRM_STS_DSCRCNT  0x0f000000  /* no. of descriptors used - 1 */
 171 #define D64_RX_FRM_STS_DATATYPE 0xf0000000      /* core-dependent data type */
 172 
 173 /*
 174  * packet headroom necessary to accommodate the largest header
 175  * in the system, (i.e TXOFF). By doing, we avoid the need to
 176  * allocate an extra buffer for the header when bridging to WL.
 177  * There is a compile time check in wlc.c which ensure that this
 178  * value is at least as big as TXOFF. This value is used in
 179  * dma_rxfill().
 180  */
 181 
 182 #define BCMEXTRAHDROOM 172
 183 
 184 #define MAXNAMEL        8       /* 8 char names */
 185 
 186 /* macros to convert between byte offsets and indexes */
 187 #define B2I(bytes, type)        ((bytes) / sizeof(type))
 188 #define I2B(index, type)        ((index) * sizeof(type))
 189 
 190 #define PCI32ADDR_HIGH          0xc0000000      /* address[31:30] */
 191 #define PCI32ADDR_HIGH_SHIFT    30      /* address[31:30] */
 192 
 193 #define PCI64ADDR_HIGH          0x80000000      /* address[63] */
 194 #define PCI64ADDR_HIGH_SHIFT    31      /* address[63] */
 195 
 196 /*
 197  * DMA Descriptor
 198  * Descriptors are only read by the hardware, never written back.
 199  */
 200 struct dma64desc {
 201         __le32 ctrl1;   /* misc control bits & bufcount */
 202         __le32 ctrl2;   /* buffer count and address extension */
 203         __le32 addrlow; /* memory address of the date buffer, bits 31:0 */
 204         __le32 addrhigh; /* memory address of the date buffer, bits 63:32 */
 205 };
 206 
 207 /* dma engine software state */
 208 struct dma_info {
 209         struct dma_pub dma; /* exported structure */
 210         char name[MAXNAMEL];    /* callers name for diag msgs */
 211 
 212         struct bcma_device *core;
 213         struct device *dmadev;
 214 
 215         /* session information for AMPDU */
 216         struct brcms_ampdu_session ampdu_session;
 217 
 218         bool dma64;     /* this dma engine is operating in 64-bit mode */
 219         bool addrext;   /* this dma engine supports DmaExtendedAddrChanges */
 220 
 221         /* 64-bit dma tx engine registers */
 222         uint d64txregbase;
 223         /* 64-bit dma rx engine registers */
 224         uint d64rxregbase;
 225         /* pointer to dma64 tx descriptor ring */
 226         struct dma64desc *txd64;
 227         /* pointer to dma64 rx descriptor ring */
 228         struct dma64desc *rxd64;
 229 
 230         u16 dmadesc_align;      /* alignment requirement for dma descriptors */
 231 
 232         u16 ntxd;               /* # tx descriptors tunable */
 233         u16 txin;               /* index of next descriptor to reclaim */
 234         u16 txout;              /* index of next descriptor to post */
 235         /* pointer to parallel array of pointers to packets */
 236         struct sk_buff **txp;
 237         /* Aligned physical address of descriptor ring */
 238         dma_addr_t txdpa;
 239         /* Original physical address of descriptor ring */
 240         dma_addr_t txdpaorig;
 241         u16 txdalign;   /* #bytes added to alloc'd mem to align txd */
 242         u32 txdalloc;   /* #bytes allocated for the ring */
 243         u32 xmtptrbase; /* When using unaligned descriptors, the ptr register
 244                          * is not just an index, it needs all 13 bits to be
 245                          * an offset from the addr register.
 246                          */
 247 
 248         u16 nrxd;       /* # rx descriptors tunable */
 249         u16 rxin;       /* index of next descriptor to reclaim */
 250         u16 rxout;      /* index of next descriptor to post */
 251         /* pointer to parallel array of pointers to packets */
 252         struct sk_buff **rxp;
 253         /* Aligned physical address of descriptor ring */
 254         dma_addr_t rxdpa;
 255         /* Original physical address of descriptor ring */
 256         dma_addr_t rxdpaorig;
 257         u16 rxdalign;   /* #bytes added to alloc'd mem to align rxd */
 258         u32 rxdalloc;   /* #bytes allocated for the ring */
 259         u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */
 260 
 261         /* tunables */
 262         unsigned int rxbufsize; /* rx buffer size in bytes, not including
 263                                  * the extra headroom
 264                                  */
 265         uint rxextrahdrroom;    /* extra rx headroom, reverseved to assist upper
 266                                  * stack, e.g. some rx pkt buffers will be
 267                                  * bridged to tx side without byte copying.
 268                                  * The extra headroom needs to be large enough
 269                                  * to fit txheader needs. Some dongle driver may
 270                                  * not need it.
 271                                  */
 272         uint nrxpost;           /* # rx buffers to keep posted */
 273         unsigned int rxoffset;  /* rxcontrol offset */
 274         /* add to get dma address of descriptor ring, low 32 bits */
 275         uint ddoffsetlow;
 276         /*   high 32 bits */
 277         uint ddoffsethigh;
 278         /* add to get dma address of data buffer, low 32 bits */
 279         uint dataoffsetlow;
 280         /*   high 32 bits */
 281         uint dataoffsethigh;
 282         /* descriptor base need to be aligned or not */
 283         bool aligndesc_4k;
 284 };
 285 
 286 /* Check for odd number of 1's */
 287 static u32 parity32(__le32 data)
 288 {
 289         /* no swap needed for counting 1's */
 290         u32 par_data = *(u32 *)&data;
 291 
 292         par_data ^= par_data >> 16;
 293         par_data ^= par_data >> 8;
 294         par_data ^= par_data >> 4;
 295         par_data ^= par_data >> 2;
 296         par_data ^= par_data >> 1;
 297 
 298         return par_data & 1;
 299 }
 300 
 301 static bool dma64_dd_parity(struct dma64desc *dd)
 302 {
 303         return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);
 304 }
 305 
 306 /* descriptor bumping functions */
 307 
 308 static uint xxd(uint x, uint n)
 309 {
 310         return x & (n - 1); /* faster than %, but n must be power of 2 */
 311 }
 312 
 313 static uint txd(struct dma_info *di, uint x)
 314 {
 315         return xxd(x, di->ntxd);
 316 }
 317 
 318 static uint rxd(struct dma_info *di, uint x)
 319 {
 320         return xxd(x, di->nrxd);
 321 }
 322 
 323 static uint nexttxd(struct dma_info *di, uint i)
 324 {
 325         return txd(di, i + 1);
 326 }
 327 
 328 static uint prevtxd(struct dma_info *di, uint i)
 329 {
 330         return txd(di, i - 1);
 331 }
 332 
 333 static uint nextrxd(struct dma_info *di, uint i)
 334 {
 335         return rxd(di, i + 1);
 336 }
 337 
 338 static uint ntxdactive(struct dma_info *di, uint h, uint t)
 339 {
 340         return txd(di, t-h);
 341 }
 342 
 343 static uint nrxdactive(struct dma_info *di, uint h, uint t)
 344 {
 345         return rxd(di, t-h);
 346 }
 347 
 348 static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)
 349 {
 350         uint dmactrlflags;
 351 
 352         if (di == NULL)
 353                 return 0;
 354 
 355         dmactrlflags = di->dma.dmactrlflags;
 356         dmactrlflags &= ~mask;
 357         dmactrlflags |= flags;
 358 
 359         /* If trying to enable parity, check if parity is actually supported */
 360         if (dmactrlflags & DMA_CTRL_PEN) {
 361                 u32 control;
 362 
 363                 control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));
 364                 bcma_write32(di->core, DMA64TXREGOFFS(di, control),
 365                       control | D64_XC_PD);
 366                 if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &
 367                     D64_XC_PD)
 368                         /* We *can* disable it so it is supported,
 369                          * restore control register
 370                          */
 371                         bcma_write32(di->core, DMA64TXREGOFFS(di, control),
 372                                      control);
 373                 else
 374                         /* Not supported, don't allow it to be enabled */
 375                         dmactrlflags &= ~DMA_CTRL_PEN;
 376         }
 377 
 378         di->dma.dmactrlflags = dmactrlflags;
 379 
 380         return dmactrlflags;
 381 }
 382 
 383 static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)
 384 {
 385         u32 w;
 386         bcma_set32(di->core, ctrl_offset, D64_XC_AE);
 387         w = bcma_read32(di->core, ctrl_offset);
 388         bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);
 389         return (w & D64_XC_AE) == D64_XC_AE;
 390 }
 391 
 392 /*
 393  * return true if this dma engine supports DmaExtendedAddrChanges,
 394  * otherwise false
 395  */
 396 static bool _dma_isaddrext(struct dma_info *di)
 397 {
 398         /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
 399 
 400         /* not all tx or rx channel are available */
 401         if (di->d64txregbase != 0) {
 402                 if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))
 403                         brcms_dbg_dma(di->core,
 404                                       "%s: DMA64 tx doesn't have AE set\n",
 405                                       di->name);
 406                 return true;
 407         } else if (di->d64rxregbase != 0) {
 408                 if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))
 409                         brcms_dbg_dma(di->core,
 410                                       "%s: DMA64 rx doesn't have AE set\n",
 411                                       di->name);
 412                 return true;
 413         }
 414 
 415         return false;
 416 }
 417 
 418 static bool _dma_descriptor_align(struct dma_info *di)
 419 {
 420         u32 addrl;
 421 
 422         /* Check to see if the descriptors need to be aligned on 4K/8K or not */
 423         if (di->d64txregbase != 0) {
 424                 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);
 425                 addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));
 426                 if (addrl != 0)
 427                         return false;
 428         } else if (di->d64rxregbase != 0) {
 429                 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);
 430                 addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));
 431                 if (addrl != 0)
 432                         return false;
 433         }
 434         return true;
 435 }
 436 
 437 /*
 438  * Descriptor table must start at the DMA hardware dictated alignment, so
 439  * allocated memory must be large enough to support this requirement.
 440  */
 441 static void *dma_alloc_consistent(struct dma_info *di, uint size,
 442                                   u16 align_bits, uint *alloced,
 443                                   dma_addr_t *pap)
 444 {
 445         if (align_bits) {
 446                 u16 align = (1 << align_bits);
 447                 if (!IS_ALIGNED(PAGE_SIZE, align))
 448                         size += align;
 449                 *alloced = size;
 450         }
 451         return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);
 452 }
 453 
 454 static
 455 u8 dma_align_sizetobits(uint size)
 456 {
 457         u8 bitpos = 0;
 458         while (size >>= 1)
 459                 bitpos++;
 460         return bitpos;
 461 }
 462 
 463 /* This function ensures that the DMA descriptor ring will not get allocated
 464  * across Page boundary. If the allocation is done across the page boundary
 465  * at the first time, then it is freed and the allocation is done at
 466  * descriptor ring size aligned location. This will ensure that the ring will
 467  * not cross page boundary
 468  */
 469 static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,
 470                            u16 *alignbits, uint *alloced,
 471                            dma_addr_t *descpa)
 472 {
 473         void *va;
 474         u32 desc_strtaddr;
 475         u32 alignbytes = 1 << *alignbits;
 476 
 477         va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);
 478 
 479         if (NULL == va)
 480                 return NULL;
 481 
 482         desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
 483         if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
 484                                                         & boundary)) {
 485                 *alignbits = dma_align_sizetobits(size);
 486                 dma_free_coherent(di->dmadev, size, va, *descpa);
 487                 va = dma_alloc_consistent(di, size, *alignbits,
 488                         alloced, descpa);
 489         }
 490         return va;
 491 }
 492 
 493 static bool dma64_alloc(struct dma_info *di, uint direction)
 494 {
 495         u16 size;
 496         uint ddlen;
 497         void *va;
 498         uint alloced = 0;
 499         u16 align;
 500         u16 align_bits;
 501 
 502         ddlen = sizeof(struct dma64desc);
 503 
 504         size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
 505         align_bits = di->dmadesc_align;
 506         align = (1 << align_bits);
 507 
 508         if (direction == DMA_TX) {
 509                 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
 510                         &alloced, &di->txdpaorig);
 511                 if (va == NULL) {
 512                         brcms_dbg_dma(di->core,
 513                                       "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
 514                                       di->name);
 515                         return false;
 516                 }
 517                 align = (1 << align_bits);
 518                 di->txd64 = (struct dma64desc *)
 519                                         roundup((unsigned long)va, align);
 520                 di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
 521                 di->txdpa = di->txdpaorig + di->txdalign;
 522                 di->txdalloc = alloced;
 523         } else {
 524                 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
 525                         &alloced, &di->rxdpaorig);
 526                 if (va == NULL) {
 527                         brcms_dbg_dma(di->core,
 528                                       "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
 529                                       di->name);
 530                         return false;
 531                 }
 532                 align = (1 << align_bits);
 533                 di->rxd64 = (struct dma64desc *)
 534                                         roundup((unsigned long)va, align);
 535                 di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
 536                 di->rxdpa = di->rxdpaorig + di->rxdalign;
 537                 di->rxdalloc = alloced;
 538         }
 539 
 540         return true;
 541 }
 542 
 543 static bool _dma_alloc(struct dma_info *di, uint direction)
 544 {
 545         return dma64_alloc(di, direction);
 546 }
 547 
 548 struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,
 549                            uint txregbase, uint rxregbase, uint ntxd, uint nrxd,
 550                            uint rxbufsize, int rxextheadroom,
 551                            uint nrxpost, uint rxoffset)
 552 {
 553         struct si_pub *sih = wlc->hw->sih;
 554         struct bcma_device *core = wlc->hw->d11core;
 555         struct dma_info *di;
 556         u8 rev = core->id.rev;
 557         uint size;
 558         struct si_info *sii = container_of(sih, struct si_info, pub);
 559 
 560         /* allocate private info structure */
 561         di = kzalloc(sizeof(struct dma_info), GFP_ATOMIC);
 562         if (di == NULL)
 563                 return NULL;
 564 
 565         di->dma64 =
 566                 ((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);
 567 
 568         /* init dma reg info */
 569         di->core = core;
 570         di->d64txregbase = txregbase;
 571         di->d64rxregbase = rxregbase;
 572 
 573         /*
 574          * Default flags (which can be changed by the driver calling
 575          * dma_ctrlflags before enable): For backwards compatibility
 576          * both Rx Overflow Continue and Parity are DISABLED.
 577          */
 578         _dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);
 579 
 580         brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d "
 581                       "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
 582                       "txregbase %u rxregbase %u\n", name, "DMA64",
 583                       di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,
 584                       rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);
 585 
 586         /* make a private copy of our callers name */
 587         strncpy(di->name, name, MAXNAMEL);
 588         di->name[MAXNAMEL - 1] = '\0';
 589 
 590         di->dmadev = core->dma_dev;
 591 
 592         /* save tunables */
 593         di->ntxd = (u16) ntxd;
 594         di->nrxd = (u16) nrxd;
 595 
 596         /* the actual dma size doesn't include the extra headroom */
 597         di->rxextrahdrroom =
 598             (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
 599         if (rxbufsize > BCMEXTRAHDROOM)
 600                 di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
 601         else
 602                 di->rxbufsize = (u16) rxbufsize;
 603 
 604         di->nrxpost = (u16) nrxpost;
 605         di->rxoffset = (u8) rxoffset;
 606 
 607         /*
 608          * figure out the DMA physical address offset for dd and data
 609          *     PCI/PCIE: they map silicon backplace address to zero
 610          *     based memory, need offset
 611          *     Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram
 612          *     swapped region for data buffer, not descriptor
 613          */
 614         di->ddoffsetlow = 0;
 615         di->dataoffsetlow = 0;
 616         /* for pci bus, add offset */
 617         if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) {
 618                 /* add offset for pcie with DMA64 bus */
 619                 di->ddoffsetlow = 0;
 620                 di->ddoffsethigh = SI_PCIE_DMA_H32;
 621         }
 622         di->dataoffsetlow = di->ddoffsetlow;
 623         di->dataoffsethigh = di->ddoffsethigh;
 624 
 625         /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
 626         if ((core->id.id == BCMA_CORE_SDIO_DEV)
 627             && ((rev > 0) && (rev <= 2)))
 628                 di->addrext = false;
 629         else if ((core->id.id == BCMA_CORE_I2S) &&
 630                  ((rev == 0) || (rev == 1)))
 631                 di->addrext = false;
 632         else
 633                 di->addrext = _dma_isaddrext(di);
 634 
 635         /* does the descriptor need to be aligned and if yes, on 4K/8K or not */
 636         di->aligndesc_4k = _dma_descriptor_align(di);
 637         if (di->aligndesc_4k) {
 638                 di->dmadesc_align = D64RINGALIGN_BITS;
 639                 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
 640                         /* for smaller dd table, HW relax alignment reqmnt */
 641                         di->dmadesc_align = D64RINGALIGN_BITS - 1;
 642         } else {
 643                 di->dmadesc_align = 4;  /* 16 byte alignment */
 644         }
 645 
 646         brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n",
 647                       di->aligndesc_4k, di->dmadesc_align);
 648 
 649         /* allocate tx packet pointer vector */
 650         if (ntxd) {
 651                 size = ntxd * sizeof(void *);
 652                 di->txp = kzalloc(size, GFP_ATOMIC);
 653                 if (di->txp == NULL)
 654                         goto fail;
 655         }
 656 
 657         /* allocate rx packet pointer vector */
 658         if (nrxd) {
 659                 size = nrxd * sizeof(void *);
 660                 di->rxp = kzalloc(size, GFP_ATOMIC);
 661                 if (di->rxp == NULL)
 662                         goto fail;
 663         }
 664 
 665         /*
 666          * allocate transmit descriptor ring, only need ntxd descriptors
 667          * but it must be aligned
 668          */
 669         if (ntxd) {
 670                 if (!_dma_alloc(di, DMA_TX))
 671                         goto fail;
 672         }
 673 
 674         /*
 675          * allocate receive descriptor ring, only need nrxd descriptors
 676          * but it must be aligned
 677          */
 678         if (nrxd) {
 679                 if (!_dma_alloc(di, DMA_RX))
 680                         goto fail;
 681         }
 682 
 683         if ((di->ddoffsetlow != 0) && !di->addrext) {
 684                 if (di->txdpa > SI_PCI_DMA_SZ) {
 685                         brcms_dbg_dma(di->core,
 686                                       "%s: txdpa 0x%x: addrext not supported\n",
 687                                       di->name, (u32)di->txdpa);
 688                         goto fail;
 689                 }
 690                 if (di->rxdpa > SI_PCI_DMA_SZ) {
 691                         brcms_dbg_dma(di->core,
 692                                       "%s: rxdpa 0x%x: addrext not supported\n",
 693                                       di->name, (u32)di->rxdpa);
 694                         goto fail;
 695                 }
 696         }
 697 
 698         /* Initialize AMPDU session */
 699         brcms_c_ampdu_reset_session(&di->ampdu_session, wlc);
 700 
 701         brcms_dbg_dma(di->core,
 702                       "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",
 703                       di->ddoffsetlow, di->ddoffsethigh,
 704                       di->dataoffsetlow, di->dataoffsethigh,
 705                       di->addrext);
 706 
 707         return (struct dma_pub *) di;
 708 
 709  fail:
 710         dma_detach((struct dma_pub *)di);
 711         return NULL;
 712 }
 713 
 714 static inline void
 715 dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,
 716              dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)
 717 {
 718         u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
 719 
 720         /* PCI bus with big(>1G) physical address, use address extension */
 721         if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {
 722                 ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
 723                 ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
 724                 ddring[outidx].ctrl1 = cpu_to_le32(*flags);
 725                 ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
 726         } else {
 727                 /* address extension for 32-bit PCI */
 728                 u32 ae;
 729 
 730                 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
 731                 pa &= ~PCI32ADDR_HIGH;
 732 
 733                 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
 734                 ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
 735                 ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
 736                 ddring[outidx].ctrl1 = cpu_to_le32(*flags);
 737                 ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
 738         }
 739         if (di->dma.dmactrlflags & DMA_CTRL_PEN) {
 740                 if (dma64_dd_parity(&ddring[outidx]))
 741                         ddring[outidx].ctrl2 =
 742                              cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);
 743         }
 744 }
 745 
 746 /* !! may be called with core in reset */
 747 void dma_detach(struct dma_pub *pub)
 748 {
 749         struct dma_info *di = container_of(pub, struct dma_info, dma);
 750 
 751         brcms_dbg_dma(di->core, "%s:\n", di->name);
 752 
 753         /* free dma descriptor rings */
 754         if (di->txd64)
 755                 dma_free_coherent(di->dmadev, di->txdalloc,
 756                                   ((s8 *)di->txd64 - di->txdalign),
 757                                   (di->txdpaorig));
 758         if (di->rxd64)
 759                 dma_free_coherent(di->dmadev, di->rxdalloc,
 760                                   ((s8 *)di->rxd64 - di->rxdalign),
 761                                   (di->rxdpaorig));
 762 
 763         /* free packet pointer vectors */
 764         kfree(di->txp);
 765         kfree(di->rxp);
 766 
 767         /* free our private info structure */
 768         kfree(di);
 769 
 770 }
 771 
 772 /* initialize descriptor table base address */
 773 static void
 774 _dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)
 775 {
 776         if (!di->aligndesc_4k) {
 777                 if (direction == DMA_TX)
 778                         di->xmtptrbase = pa;
 779                 else
 780                         di->rcvptrbase = pa;
 781         }
 782 
 783         if ((di->ddoffsetlow == 0)
 784             || !(pa & PCI32ADDR_HIGH)) {
 785                 if (direction == DMA_TX) {
 786                         bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
 787                                      pa + di->ddoffsetlow);
 788                         bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
 789                                      di->ddoffsethigh);
 790                 } else {
 791                         bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
 792                                      pa + di->ddoffsetlow);
 793                         bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
 794                                      di->ddoffsethigh);
 795                 }
 796         } else {
 797                 /* DMA64 32bits address extension */
 798                 u32 ae;
 799 
 800                 /* shift the high bit(s) from pa to ae */
 801                 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
 802                 pa &= ~PCI32ADDR_HIGH;
 803 
 804                 if (direction == DMA_TX) {
 805                         bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
 806                                      pa + di->ddoffsetlow);
 807                         bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
 808                                      di->ddoffsethigh);
 809                         bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),
 810                                        D64_XC_AE, (ae << D64_XC_AE_SHIFT));
 811                 } else {
 812                         bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
 813                                      pa + di->ddoffsetlow);
 814                         bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
 815                                      di->ddoffsethigh);
 816                         bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),
 817                                        D64_RC_AE, (ae << D64_RC_AE_SHIFT));
 818                 }
 819         }
 820 }
 821 
 822 static void _dma_rxenable(struct dma_info *di)
 823 {
 824         uint dmactrlflags = di->dma.dmactrlflags;
 825         u32 control;
 826 
 827         brcms_dbg_dma(di->core, "%s:\n", di->name);
 828 
 829         control = D64_RC_RE | (bcma_read32(di->core,
 830                                            DMA64RXREGOFFS(di, control)) &
 831                                D64_RC_AE);
 832 
 833         if ((dmactrlflags & DMA_CTRL_PEN) == 0)
 834                 control |= D64_RC_PD;
 835 
 836         if (dmactrlflags & DMA_CTRL_ROC)
 837                 control |= D64_RC_OC;
 838 
 839         bcma_write32(di->core, DMA64RXREGOFFS(di, control),
 840                 ((di->rxoffset << D64_RC_RO_SHIFT) | control));
 841 }
 842 
 843 void dma_rxinit(struct dma_pub *pub)
 844 {
 845         struct dma_info *di = container_of(pub, struct dma_info, dma);
 846 
 847         brcms_dbg_dma(di->core, "%s:\n", di->name);
 848 
 849         if (di->nrxd == 0)
 850                 return;
 851 
 852         di->rxin = di->rxout = 0;
 853 
 854         /* clear rx descriptor ring */
 855         memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));
 856 
 857         /* DMA engine with out alignment requirement requires table to be inited
 858          * before enabling the engine
 859          */
 860         if (!di->aligndesc_4k)
 861                 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
 862 
 863         _dma_rxenable(di);
 864 
 865         if (di->aligndesc_4k)
 866                 _dma_ddtable_init(di, DMA_RX, di->rxdpa);
 867 }
 868 
 869 static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)
 870 {
 871         uint i, curr;
 872         struct sk_buff *rxp;
 873         dma_addr_t pa;
 874 
 875         i = di->rxin;
 876 
 877         /* return if no packets posted */
 878         if (i == di->rxout)
 879                 return NULL;
 880 
 881         curr =
 882             B2I(((bcma_read32(di->core,
 883                               DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -
 884                  di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);
 885 
 886         /* ignore curr if forceall */
 887         if (!forceall && (i == curr))
 888                 return NULL;
 889 
 890         /* get the packet pointer that corresponds to the rx descriptor */
 891         rxp = di->rxp[i];
 892         di->rxp[i] = NULL;
 893 
 894         pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;
 895 
 896         /* clear this packet from the descriptor ring */
 897         dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);
 898 
 899         di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);
 900         di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
 901 
 902         di->rxin = nextrxd(di, i);
 903 
 904         return rxp;
 905 }
 906 
 907 static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)
 908 {
 909         if (di->nrxd == 0)
 910                 return NULL;
 911 
 912         return dma64_getnextrxp(di, forceall);
 913 }
 914 
 915 /*
 916  * !! rx entry routine
 917  * returns the number packages in the next frame, or 0 if there are no more
 918  *   if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is
 919  *   supported with pkts chain
 920  *   otherwise, it's treated as giant pkt and will be tossed.
 921  *   The DMA scattering starts with normal DMA header, followed by first
 922  *   buffer data. After it reaches the max size of buffer, the data continues
 923  *   in next DMA descriptor buffer WITHOUT DMA header
 924  */
 925 int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)
 926 {
 927         struct dma_info *di = container_of(pub, struct dma_info, dma);
 928         struct sk_buff_head dma_frames;
 929         struct sk_buff *p, *next;
 930         uint len;
 931         uint pkt_len;
 932         int resid = 0;
 933         int pktcnt = 1;
 934 
 935         skb_queue_head_init(&dma_frames);
 936  next_frame:
 937         p = _dma_getnextrxp(di, false);
 938         if (p == NULL)
 939                 return 0;
 940 
 941         len = le16_to_cpu(*(__le16 *) (p->data));
 942         brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len);
 943         dma_spin_for_len(len, p);
 944 
 945         /* set actual length */
 946         pkt_len = min((di->rxoffset + len), di->rxbufsize);
 947         __skb_trim(p, pkt_len);
 948         skb_queue_tail(&dma_frames, p);
 949         resid = len - (di->rxbufsize - di->rxoffset);
 950 
 951         /* check for single or multi-buffer rx */
 952         if (resid > 0) {
 953                 while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
 954                         pkt_len = min_t(uint, resid, di->rxbufsize);
 955                         __skb_trim(p, pkt_len);
 956                         skb_queue_tail(&dma_frames, p);
 957                         resid -= di->rxbufsize;
 958                         pktcnt++;
 959                 }
 960 
 961 #ifdef DEBUG
 962                 if (resid > 0) {
 963                         uint cur;
 964                         cur =
 965                             B2I(((bcma_read32(di->core,
 966                                               DMA64RXREGOFFS(di, status0)) &
 967                                   D64_RS0_CD_MASK) - di->rcvptrbase) &
 968                                 D64_RS0_CD_MASK, struct dma64desc);
 969                         brcms_dbg_dma(di->core,
 970                                       "rxin %d rxout %d, hw_curr %d\n",
 971                                       di->rxin, di->rxout, cur);
 972                 }
 973 #endif                          /* DEBUG */
 974 
 975                 if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
 976                         brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n",
 977                                       di->name, len);
 978                         skb_queue_walk_safe(&dma_frames, p, next) {
 979                                 skb_unlink(p, &dma_frames);
 980                                 brcmu_pkt_buf_free_skb(p);
 981                         }
 982                         di->dma.rxgiants++;
 983                         pktcnt = 1;
 984                         goto next_frame;
 985                 }
 986         }
 987 
 988         skb_queue_splice_tail(&dma_frames, skb_list);
 989         return pktcnt;
 990 }
 991 
 992 static bool dma64_rxidle(struct dma_info *di)
 993 {
 994         brcms_dbg_dma(di->core, "%s:\n", di->name);
 995 
 996         if (di->nrxd == 0)
 997                 return true;
 998 
 999         return ((bcma_read32(di->core,
1000                              DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==
1001                 (bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &
1002                  D64_RS0_CD_MASK));
1003 }
1004 
1005 static bool dma64_txidle(struct dma_info *di)
1006 {
1007         if (di->ntxd == 0)
1008                 return true;
1009 
1010         return ((bcma_read32(di->core,
1011                              DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) ==
1012                 (bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) &
1013                  D64_XS0_CD_MASK));
1014 }
1015 
1016 /*
1017  * post receive buffers
1018  *  Return false if refill failed completely or dma mapping failed. The ring
1019  *  is empty, which will stall the rx dma and user might want to call rxfill
1020  *  again asap. This is unlikely to happen on a memory-rich NIC, but often on
1021  *  memory-constrained dongle.
1022  */
1023 bool dma_rxfill(struct dma_pub *pub)
1024 {
1025         struct dma_info *di = container_of(pub, struct dma_info, dma);
1026         struct sk_buff *p;
1027         u16 rxin, rxout;
1028         u32 flags = 0;
1029         uint n;
1030         uint i;
1031         dma_addr_t pa;
1032         uint extra_offset = 0;
1033         bool ring_empty;
1034 
1035         ring_empty = false;
1036 
1037         /*
1038          * Determine how many receive buffers we're lacking
1039          * from the full complement, allocate, initialize,
1040          * and post them, then update the chip rx lastdscr.
1041          */
1042 
1043         rxin = di->rxin;
1044         rxout = di->rxout;
1045 
1046         n = di->nrxpost - nrxdactive(di, rxin, rxout);
1047 
1048         brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n);
1049 
1050         if (di->rxbufsize > BCMEXTRAHDROOM)
1051                 extra_offset = di->rxextrahdrroom;
1052 
1053         for (i = 0; i < n; i++) {
1054                 /*
1055                  * the di->rxbufsize doesn't include the extra headroom,
1056                  * we need to add it to the size to be allocated
1057                  */
1058                 p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);
1059 
1060                 if (p == NULL) {
1061                         brcms_dbg_dma(di->core, "%s: out of rxbufs\n",
1062                                       di->name);
1063                         if (i == 0 && dma64_rxidle(di)) {
1064                                 brcms_dbg_dma(di->core, "%s: ring is empty !\n",
1065                                               di->name);
1066                                 ring_empty = true;
1067                         }
1068                         di->dma.rxnobuf++;
1069                         break;
1070                 }
1071                 /* reserve an extra headroom, if applicable */
1072                 if (extra_offset)
1073                         skb_pull(p, extra_offset);
1074 
1075                 /* Do a cached write instead of uncached write since DMA_MAP
1076                  * will flush the cache.
1077                  */
1078                 *(u32 *) (p->data) = 0;
1079 
1080                 pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,
1081                                     DMA_FROM_DEVICE);
1082                 if (dma_mapping_error(di->dmadev, pa)) {
1083                         brcmu_pkt_buf_free_skb(p);
1084                         return false;
1085                 }
1086 
1087                 /* save the free packet pointer */
1088                 di->rxp[rxout] = p;
1089 
1090                 /* reset flags for each descriptor */
1091                 flags = 0;
1092                 if (rxout == (di->nrxd - 1))
1093                         flags = D64_CTRL1_EOT;
1094 
1095                 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
1096                              di->rxbufsize);
1097                 rxout = nextrxd(di, rxout);
1098         }
1099 
1100         di->rxout = rxout;
1101 
1102         /* update the chip lastdscr pointer */
1103         bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),
1104               di->rcvptrbase + I2B(rxout, struct dma64desc));
1105 
1106         return ring_empty;
1107 }
1108 
1109 void dma_rxreclaim(struct dma_pub *pub)
1110 {
1111         struct dma_info *di = container_of(pub, struct dma_info, dma);
1112         struct sk_buff *p;
1113 
1114         brcms_dbg_dma(di->core, "%s:\n", di->name);
1115 
1116         while ((p = _dma_getnextrxp(di, true)))
1117                 brcmu_pkt_buf_free_skb(p);
1118 }
1119 
1120 void dma_counterreset(struct dma_pub *pub)
1121 {
1122         /* reset all software counters */
1123         pub->rxgiants = 0;
1124         pub->rxnobuf = 0;
1125         pub->txnobuf = 0;
1126 }
1127 
1128 /* get the address of the var in order to change later */
1129 unsigned long dma_getvar(struct dma_pub *pub, const char *name)
1130 {
1131         struct dma_info *di = container_of(pub, struct dma_info, dma);
1132 
1133         if (!strcmp(name, "&txavail"))
1134                 return (unsigned long)&(di->dma.txavail);
1135         return 0;
1136 }
1137 
1138 /* 64-bit DMA functions */
1139 
1140 void dma_txinit(struct dma_pub *pub)
1141 {
1142         struct dma_info *di = container_of(pub, struct dma_info, dma);
1143         u32 control = D64_XC_XE;
1144 
1145         brcms_dbg_dma(di->core, "%s:\n", di->name);
1146 
1147         if (di->ntxd == 0)
1148                 return;
1149 
1150         di->txin = di->txout = 0;
1151         di->dma.txavail = di->ntxd - 1;
1152 
1153         /* clear tx descriptor ring */
1154         memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));
1155 
1156         /* DMA engine with out alignment requirement requires table to be inited
1157          * before enabling the engine
1158          */
1159         if (!di->aligndesc_4k)
1160                 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1161 
1162         if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)
1163                 control |= D64_XC_PD;
1164         bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);
1165 
1166         /* DMA engine with alignment requirement requires table to be inited
1167          * before enabling the engine
1168          */
1169         if (di->aligndesc_4k)
1170                 _dma_ddtable_init(di, DMA_TX, di->txdpa);
1171 }
1172 
1173 void dma_txsuspend(struct dma_pub *pub)
1174 {
1175         struct dma_info *di = container_of(pub, struct dma_info, dma);
1176 
1177         brcms_dbg_dma(di->core, "%s:\n", di->name);
1178 
1179         if (di->ntxd == 0)
1180                 return;
1181 
1182         bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1183 }
1184 
1185 void dma_txresume(struct dma_pub *pub)
1186 {
1187         struct dma_info *di = container_of(pub, struct dma_info, dma);
1188 
1189         brcms_dbg_dma(di->core, "%s:\n", di->name);
1190 
1191         if (di->ntxd == 0)
1192                 return;
1193 
1194         bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);
1195 }
1196 
1197 bool dma_txsuspended(struct dma_pub *pub)
1198 {
1199         struct dma_info *di = container_of(pub, struct dma_info, dma);
1200 
1201         return (di->ntxd == 0) ||
1202                ((bcma_read32(di->core,
1203                              DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==
1204                 D64_XC_SE);
1205 }
1206 
1207 void dma_txreclaim(struct dma_pub *pub, enum txd_range range)
1208 {
1209         struct dma_info *di = container_of(pub, struct dma_info, dma);
1210         struct sk_buff *p;
1211 
1212         brcms_dbg_dma(di->core, "%s: %s\n",
1213                       di->name,
1214                       range == DMA_RANGE_ALL ? "all" :
1215                       range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1216                       "transferred");
1217 
1218         if (di->txin == di->txout)
1219                 return;
1220 
1221         while ((p = dma_getnexttxp(pub, range))) {
1222                 /* For unframed data, we don't have any packets to free */
1223                 if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))
1224                         brcmu_pkt_buf_free_skb(p);
1225         }
1226 }
1227 
1228 bool dma_txreset(struct dma_pub *pub)
1229 {
1230         struct dma_info *di = container_of(pub, struct dma_info, dma);
1231         u32 status;
1232 
1233         if (di->ntxd == 0)
1234                 return true;
1235 
1236         /* suspend tx DMA first */
1237         bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1238         SPINWAIT(((status =
1239                    (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1240                     D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&
1241                   (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),
1242                  10000);
1243 
1244         bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);
1245         SPINWAIT(((status =
1246                    (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1247                     D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);
1248 
1249         /* wait for the last transaction to complete */
1250         udelay(300);
1251 
1252         return status == D64_XS0_XS_DISABLED;
1253 }
1254 
1255 bool dma_rxreset(struct dma_pub *pub)
1256 {
1257         struct dma_info *di = container_of(pub, struct dma_info, dma);
1258         u32 status;
1259 
1260         if (di->nrxd == 0)
1261                 return true;
1262 
1263         bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);
1264         SPINWAIT(((status =
1265                    (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &
1266                     D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);
1267 
1268         return status == D64_RS0_RS_DISABLED;
1269 }
1270 
1271 static void dma_txenq(struct dma_info *di, struct sk_buff *p)
1272 {
1273         unsigned char *data;
1274         uint len;
1275         u16 txout;
1276         u32 flags = 0;
1277         dma_addr_t pa;
1278 
1279         txout = di->txout;
1280 
1281         if (WARN_ON(nexttxd(di, txout) == di->txin))
1282                 return;
1283 
1284         /*
1285          * obtain and initialize transmit descriptor entry.
1286          */
1287         data = p->data;
1288         len = p->len;
1289 
1290         /* get physical address of buffer start */
1291         pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);
1292         /* if mapping failed, free skb */
1293         if (dma_mapping_error(di->dmadev, pa)) {
1294                 brcmu_pkt_buf_free_skb(p);
1295                 return;
1296         }
1297         /* With a DMA segment list, Descriptor table is filled
1298          * using the segment list instead of looping over
1299          * buffers in multi-chain DMA. Therefore, EOF for SGLIST
1300          * is when end of segment list is reached.
1301          */
1302         flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;
1303         if (txout == (di->ntxd - 1))
1304                 flags |= D64_CTRL1_EOT;
1305 
1306         dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1307 
1308         txout = nexttxd(di, txout);
1309 
1310         /* save the packet */
1311         di->txp[prevtxd(di, txout)] = p;
1312 
1313         /* bump the tx descriptor index */
1314         di->txout = txout;
1315 }
1316 
1317 static void ampdu_finalize(struct dma_info *di)
1318 {
1319         struct brcms_ampdu_session *session = &di->ampdu_session;
1320         struct sk_buff *p;
1321 
1322         trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev,
1323                                   session->max_ampdu_len,
1324                                   session->max_ampdu_frames,
1325                                   session->ampdu_len,
1326                                   skb_queue_len(&session->skb_list),
1327                                   session->dma_len);
1328 
1329         if (WARN_ON(skb_queue_empty(&session->skb_list)))
1330                 return;
1331 
1332         brcms_c_ampdu_finalize(session);
1333 
1334         while (!skb_queue_empty(&session->skb_list)) {
1335                 p = skb_dequeue(&session->skb_list);
1336                 dma_txenq(di, p);
1337         }
1338 
1339         bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1340                      di->xmtptrbase + I2B(di->txout, struct dma64desc));
1341         brcms_c_ampdu_reset_session(session, session->wlc);
1342 }
1343 
1344 static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p)
1345 {
1346         struct brcms_ampdu_session *session = &di->ampdu_session;
1347         int ret;
1348 
1349         ret = brcms_c_ampdu_add_frame(session, p);
1350         if (ret == -ENOSPC) {
1351                 /*
1352                  * AMPDU cannot accomodate this frame. Close out the in-
1353                  * progress AMPDU session and start a new one.
1354                  */
1355                 ampdu_finalize(di);
1356                 ret = brcms_c_ampdu_add_frame(session, p);
1357         }
1358 
1359         WARN_ON(ret);
1360 }
1361 
1362 /* Update count of available tx descriptors based on current DMA state */
1363 static void dma_update_txavail(struct dma_info *di)
1364 {
1365         /*
1366          * Available space is number of descriptors less the number of
1367          * active descriptors and the number of queued AMPDU frames.
1368          */
1369         di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) -
1370                           skb_queue_len(&di->ampdu_session.skb_list) - 1;
1371 }
1372 
1373 /*
1374  * !! tx entry routine
1375  * WARNING: call must check the return value for error.
1376  *   the error(toss frames) could be fatal and cause many subsequent hard
1377  *   to debug problems
1378  */
1379 int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub,
1380                struct sk_buff *p)
1381 {
1382         struct dma_info *di = container_of(pub, struct dma_info, dma);
1383         struct brcms_ampdu_session *session = &di->ampdu_session;
1384         struct ieee80211_tx_info *tx_info;
1385         bool is_ampdu;
1386 
1387         /* no use to transmit a zero length packet */
1388         if (p->len == 0)
1389                 return 0;
1390 
1391         /* return nonzero if out of tx descriptors */
1392         if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin)
1393                 goto outoftxd;
1394 
1395         tx_info = IEEE80211_SKB_CB(p);
1396         is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU;
1397         if (is_ampdu)
1398                 prep_ampdu_frame(di, p);
1399         else
1400                 dma_txenq(di, p);
1401 
1402         /* tx flow control */
1403         dma_update_txavail(di);
1404 
1405         /* kick the chip */
1406         if (is_ampdu) {
1407                 /*
1408                  * Start sending data if we've got a full AMPDU, there's
1409                  * no more space in the DMA ring, or the ring isn't
1410                  * currently transmitting.
1411                  */
1412                 if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames ||
1413                     di->dma.txavail == 0 || dma64_txidle(di))
1414                         ampdu_finalize(di);
1415         } else {
1416                 bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1417                              di->xmtptrbase + I2B(di->txout, struct dma64desc));
1418         }
1419 
1420         return 0;
1421 
1422  outoftxd:
1423         brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name);
1424         brcmu_pkt_buf_free_skb(p);
1425         di->dma.txavail = 0;
1426         di->dma.txnobuf++;
1427         return -ENOSPC;
1428 }
1429 
1430 void dma_txflush(struct dma_pub *pub)
1431 {
1432         struct dma_info *di = container_of(pub, struct dma_info, dma);
1433         struct brcms_ampdu_session *session = &di->ampdu_session;
1434 
1435         if (!skb_queue_empty(&session->skb_list))
1436                 ampdu_finalize(di);
1437 }
1438 
1439 int dma_txpending(struct dma_pub *pub)
1440 {
1441         struct dma_info *di = container_of(pub, struct dma_info, dma);
1442         return ntxdactive(di, di->txin, di->txout);
1443 }
1444 
1445 /*
1446  * If we have an active AMPDU session and are not transmitting,
1447  * this function will force tx to start.
1448  */
1449 void dma_kick_tx(struct dma_pub *pub)
1450 {
1451         struct dma_info *di = container_of(pub, struct dma_info, dma);
1452         struct brcms_ampdu_session *session = &di->ampdu_session;
1453 
1454         if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di))
1455                 ampdu_finalize(di);
1456 }
1457 
1458 /*
1459  * Reclaim next completed txd (txds if using chained buffers) in the range
1460  * specified and return associated packet.
1461  * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1462  * transmitted as noted by the hardware "CurrDescr" pointer.
1463  * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be
1464  * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
1465  * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1466  * return associated packet regardless of the value of hardware pointers.
1467  */
1468 struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)
1469 {
1470         struct dma_info *di = container_of(pub, struct dma_info, dma);
1471         u16 start, end, i;
1472         u16 active_desc;
1473         struct sk_buff *txp;
1474 
1475         brcms_dbg_dma(di->core, "%s: %s\n",
1476                       di->name,
1477                       range == DMA_RANGE_ALL ? "all" :
1478                       range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1479                       "transferred");
1480 
1481         if (di->ntxd == 0)
1482                 return NULL;
1483 
1484         txp = NULL;
1485 
1486         start = di->txin;
1487         if (range == DMA_RANGE_ALL)
1488                 end = di->txout;
1489         else {
1490                 end = (u16) (B2I(((bcma_read32(di->core,
1491                                                DMA64TXREGOFFS(di, status0)) &
1492                                    D64_XS0_CD_MASK) - di->xmtptrbase) &
1493                                  D64_XS0_CD_MASK, struct dma64desc));
1494 
1495                 if (range == DMA_RANGE_TRANSFERED) {
1496                         active_desc =
1497                                 (u16)(bcma_read32(di->core,
1498                                                   DMA64TXREGOFFS(di, status1)) &
1499                                       D64_XS1_AD_MASK);
1500                         active_desc =
1501                             (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
1502                         active_desc = B2I(active_desc, struct dma64desc);
1503                         if (end != active_desc)
1504                                 end = prevtxd(di, active_desc);
1505                 }
1506         }
1507 
1508         if ((start == 0) && (end > di->txout))
1509                 goto bogus;
1510 
1511         for (i = start; i != end && !txp; i = nexttxd(di, i)) {
1512                 dma_addr_t pa;
1513                 uint size;
1514 
1515                 pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;
1516 
1517                 size =
1518                     (le32_to_cpu(di->txd64[i].ctrl2) &
1519                      D64_CTRL2_BC_MASK);
1520 
1521                 di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);
1522                 di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
1523 
1524                 txp = di->txp[i];
1525                 di->txp[i] = NULL;
1526 
1527                 dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);
1528         }
1529 
1530         di->txin = i;
1531 
1532         /* tx flow control */
1533         dma_update_txavail(di);
1534 
1535         return txp;
1536 
1537  bogus:
1538         brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n",
1539                       start, end, di->txout);
1540         return NULL;
1541 }
1542 
1543 /*
1544  * Mac80211 initiated actions sometimes require packets in the DMA queue to be
1545  * modified. The modified portion of the packet is not under control of the DMA
1546  * engine. This function calls a caller-supplied function for each packet in
1547  * the caller specified dma chain.
1548  */
1549 void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)
1550                       (void *pkt, void *arg_a), void *arg_a)
1551 {
1552         struct dma_info *di = container_of(dmah, struct dma_info, dma);
1553         uint i =   di->txin;
1554         uint end = di->txout;
1555         struct sk_buff *skb;
1556         struct ieee80211_tx_info *tx_info;
1557 
1558         while (i != end) {
1559                 skb = di->txp[i];
1560                 if (skb != NULL) {
1561                         tx_info = (struct ieee80211_tx_info *)skb->cb;
1562                         (callback_fnc)(tx_info, arg_a);
1563                 }
1564                 i = nexttxd(di, i);
1565         }
1566 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c

DEFINITIONS
