root/include/linux/kfifo.h

INCLUDED FROM

DEFINITIONS

1. __kfifo_uint_must_check_helper
2. __kfifo_int_must_check_helper

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * A generic kernel FIFO implementation
   4  *
   5  * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net>
   6  */
   7 
   8 #ifndef _LINUX_KFIFO_H
   9 #define _LINUX_KFIFO_H
  10 
  11 /*
  12  * How to porting drivers to the new generic FIFO API:
  13  *
  14  * - Modify the declaration of the "struct kfifo *" object into a
  15  *   in-place "struct kfifo" object
  16  * - Init the in-place object with kfifo_alloc() or kfifo_init()
  17  *   Note: The address of the in-place "struct kfifo" object must be
  18  *   passed as the first argument to this functions
  19  * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
  20  *   into kfifo_out
  21  * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
  22  *   into kfifo_out_spinlocked
  23  *   Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
  24  *   must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
  25  *   as the last parameter
  26  * - The formerly __kfifo_* functions are renamed into kfifo_*
  27  */
  28 
  29 /*
  30  * Note about locking: There is no locking required until only one reader
  31  * and one writer is using the fifo and no kfifo_reset() will be called.
  32  * kfifo_reset_out() can be safely used, until it will be only called
  33  * in the reader thread.
  34  * For multiple writer and one reader there is only a need to lock the writer.
  35  * And vice versa for only one writer and multiple reader there is only a need
  36  * to lock the reader.
  37  */
  38 
  39 #include <linux/kernel.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/stddef.h>
  42 #include <linux/scatterlist.h>
  43 
  44 struct __kfifo {
  45         unsigned int    in;
  46         unsigned int    out;
  47         unsigned int    mask;
  48         unsigned int    esize;
  49         void            *data;
  50 };
  51 
  52 #define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
  53         union { \
  54                 struct __kfifo  kfifo; \
  55                 datatype        *type; \
  56                 const datatype  *const_type; \
  57                 char            (*rectype)[recsize]; \
  58                 ptrtype         *ptr; \
  59                 ptrtype const   *ptr_const; \
  60         }
  61 
  62 #define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
  63 { \
  64         __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
  65         type            buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
  66 }
  67 
  68 #define STRUCT_KFIFO(type, size) \
  69         struct __STRUCT_KFIFO(type, size, 0, type)
  70 
  71 #define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
  72 { \
  73         __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
  74         type            buf[0]; \
  75 }
  76 
  77 #define STRUCT_KFIFO_PTR(type) \
  78         struct __STRUCT_KFIFO_PTR(type, 0, type)
  79 
  80 /*
  81  * define compatibility "struct kfifo" for dynamic allocated fifos
  82  */
  83 struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
  84 
  85 #define STRUCT_KFIFO_REC_1(size) \
  86         struct __STRUCT_KFIFO(unsigned char, size, 1, void)
  87 
  88 #define STRUCT_KFIFO_REC_2(size) \
  89         struct __STRUCT_KFIFO(unsigned char, size, 2, void)
  90 
  91 /*
  92  * define kfifo_rec types
  93  */
  94 struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
  95 struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
  96 
  97 /*
  98  * helper macro to distinguish between real in place fifo where the fifo
  99  * array is a part of the structure and the fifo type where the array is
 100  * outside of the fifo structure.
 101  */
 102 #define __is_kfifo_ptr(fifo) \
 103         (sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
 104 
 105 /**
 106  * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
 107  * @fifo: name of the declared fifo
 108  * @type: type of the fifo elements
 109  */
 110 #define DECLARE_KFIFO_PTR(fifo, type)   STRUCT_KFIFO_PTR(type) fifo
 111 
 112 /**
 113  * DECLARE_KFIFO - macro to declare a fifo object
 114  * @fifo: name of the declared fifo
 115  * @type: type of the fifo elements
 116  * @size: the number of elements in the fifo, this must be a power of 2
 117  */
 118 #define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo
 119 
 120 /**
 121  * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
 122  * @fifo: name of the declared fifo datatype
 123  */
 124 #define INIT_KFIFO(fifo) \
 125 (void)({ \
 126         typeof(&(fifo)) __tmp = &(fifo); \
 127         struct __kfifo *__kfifo = &__tmp->kfifo; \
 128         __kfifo->in = 0; \
 129         __kfifo->out = 0; \
 130         __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
 131         __kfifo->esize = sizeof(*__tmp->buf); \
 132         __kfifo->data = __is_kfifo_ptr(__tmp) ?  NULL : __tmp->buf; \
 133 })
 134 
 135 /**
 136  * DEFINE_KFIFO - macro to define and initialize a fifo
 137  * @fifo: name of the declared fifo datatype
 138  * @type: type of the fifo elements
 139  * @size: the number of elements in the fifo, this must be a power of 2
 140  *
 141  * Note: the macro can be used for global and local fifo data type variables.
 142  */
 143 #define DEFINE_KFIFO(fifo, type, size) \
 144         DECLARE_KFIFO(fifo, type, size) = \
 145         (typeof(fifo)) { \
 146                 { \
 147                         { \
 148                         .in     = 0, \
 149                         .out    = 0, \
 150                         .mask   = __is_kfifo_ptr(&(fifo)) ? \
 151                                   0 : \
 152                                   ARRAY_SIZE((fifo).buf) - 1, \
 153                         .esize  = sizeof(*(fifo).buf), \
 154                         .data   = __is_kfifo_ptr(&(fifo)) ? \
 155                                 NULL : \
 156                                 (fifo).buf, \
 157                         } \
 158                 } \
 159         }
 160 
 161 
 162 static inline unsigned int __must_check
 163 __kfifo_uint_must_check_helper(unsigned int val)
 164 {
 165         return val;
 166 }
 167 
 168 static inline int __must_check
 169 __kfifo_int_must_check_helper(int val)
 170 {
 171         return val;
 172 }
 173 
 174 /**
 175  * kfifo_initialized - Check if the fifo is initialized
 176  * @fifo: address of the fifo to check
 177  *
 178  * Return %true if fifo is initialized, otherwise %false.
 179  * Assumes the fifo was 0 before.
 180  */
 181 #define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
 182 
 183 /**
 184  * kfifo_esize - returns the size of the element managed by the fifo
 185  * @fifo: address of the fifo to be used
 186  */
 187 #define kfifo_esize(fifo)       ((fifo)->kfifo.esize)
 188 
 189 /**
 190  * kfifo_recsize - returns the size of the record length field
 191  * @fifo: address of the fifo to be used
 192  */
 193 #define kfifo_recsize(fifo)     (sizeof(*(fifo)->rectype))
 194 
 195 /**
 196  * kfifo_size - returns the size of the fifo in elements
 197  * @fifo: address of the fifo to be used
 198  */
 199 #define kfifo_size(fifo)        ((fifo)->kfifo.mask + 1)
 200 
 201 /**
 202  * kfifo_reset - removes the entire fifo content
 203  * @fifo: address of the fifo to be used
 204  *
 205  * Note: usage of kfifo_reset() is dangerous. It should be only called when the
 206  * fifo is exclusived locked or when it is secured that no other thread is
 207  * accessing the fifo.
 208  */
 209 #define kfifo_reset(fifo) \
 210 (void)({ \
 211         typeof((fifo) + 1) __tmp = (fifo); \
 212         __tmp->kfifo.in = __tmp->kfifo.out = 0; \
 213 })
 214 
 215 /**
 216  * kfifo_reset_out - skip fifo content
 217  * @fifo: address of the fifo to be used
 218  *
 219  * Note: The usage of kfifo_reset_out() is safe until it will be only called
 220  * from the reader thread and there is only one concurrent reader. Otherwise
 221  * it is dangerous and must be handled in the same way as kfifo_reset().
 222  */
 223 #define kfifo_reset_out(fifo)   \
 224 (void)({ \
 225         typeof((fifo) + 1) __tmp = (fifo); \
 226         __tmp->kfifo.out = __tmp->kfifo.in; \
 227 })
 228 
 229 /**
 230  * kfifo_len - returns the number of used elements in the fifo
 231  * @fifo: address of the fifo to be used
 232  */
 233 #define kfifo_len(fifo) \
 234 ({ \
 235         typeof((fifo) + 1) __tmpl = (fifo); \
 236         __tmpl->kfifo.in - __tmpl->kfifo.out; \
 237 })
 238 
 239 /**
 240  * kfifo_is_empty - returns true if the fifo is empty
 241  * @fifo: address of the fifo to be used
 242  */
 243 #define kfifo_is_empty(fifo) \
 244 ({ \
 245         typeof((fifo) + 1) __tmpq = (fifo); \
 246         __tmpq->kfifo.in == __tmpq->kfifo.out; \
 247 })
 248 
 249 /**
 250  * kfifo_is_full - returns true if the fifo is full
 251  * @fifo: address of the fifo to be used
 252  */
 253 #define kfifo_is_full(fifo) \
 254 ({ \
 255         typeof((fifo) + 1) __tmpq = (fifo); \
 256         kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
 257 })
 258 
 259 /**
 260  * kfifo_avail - returns the number of unused elements in the fifo
 261  * @fifo: address of the fifo to be used
 262  */
 263 #define kfifo_avail(fifo) \
 264 __kfifo_uint_must_check_helper( \
 265 ({ \
 266         typeof((fifo) + 1) __tmpq = (fifo); \
 267         const size_t __recsize = sizeof(*__tmpq->rectype); \
 268         unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
 269         (__recsize) ? ((__avail <= __recsize) ? 0 : \
 270         __kfifo_max_r(__avail - __recsize, __recsize)) : \
 271         __avail; \
 272 }) \
 273 )
 274 
 275 /**
 276  * kfifo_skip - skip output data
 277  * @fifo: address of the fifo to be used
 278  */
 279 #define kfifo_skip(fifo) \
 280 (void)({ \
 281         typeof((fifo) + 1) __tmp = (fifo); \
 282         const size_t __recsize = sizeof(*__tmp->rectype); \
 283         struct __kfifo *__kfifo = &__tmp->kfifo; \
 284         if (__recsize) \
 285                 __kfifo_skip_r(__kfifo, __recsize); \
 286         else \
 287                 __kfifo->out++; \
 288 })
 289 
 290 /**
 291  * kfifo_peek_len - gets the size of the next fifo record
 292  * @fifo: address of the fifo to be used
 293  *
 294  * This function returns the size of the next fifo record in number of bytes.
 295  */
 296 #define kfifo_peek_len(fifo) \
 297 __kfifo_uint_must_check_helper( \
 298 ({ \
 299         typeof((fifo) + 1) __tmp = (fifo); \
 300         const size_t __recsize = sizeof(*__tmp->rectype); \
 301         struct __kfifo *__kfifo = &__tmp->kfifo; \
 302         (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
 303         __kfifo_len_r(__kfifo, __recsize); \
 304 }) \
 305 )
 306 
 307 /**
 308  * kfifo_alloc - dynamically allocates a new fifo buffer
 309  * @fifo: pointer to the fifo
 310  * @size: the number of elements in the fifo, this must be a power of 2
 311  * @gfp_mask: get_free_pages mask, passed to kmalloc()
 312  *
 313  * This macro dynamically allocates a new fifo buffer.
 314  *
 315  * The number of elements will be rounded-up to a power of 2.
 316  * The fifo will be release with kfifo_free().
 317  * Return 0 if no error, otherwise an error code.
 318  */
 319 #define kfifo_alloc(fifo, size, gfp_mask) \
 320 __kfifo_int_must_check_helper( \
 321 ({ \
 322         typeof((fifo) + 1) __tmp = (fifo); \
 323         struct __kfifo *__kfifo = &__tmp->kfifo; \
 324         __is_kfifo_ptr(__tmp) ? \
 325         __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
 326         -EINVAL; \
 327 }) \
 328 )
 329 
 330 /**
 331  * kfifo_free - frees the fifo
 332  * @fifo: the fifo to be freed
 333  */
 334 #define kfifo_free(fifo) \
 335 ({ \
 336         typeof((fifo) + 1) __tmp = (fifo); \
 337         struct __kfifo *__kfifo = &__tmp->kfifo; \
 338         if (__is_kfifo_ptr(__tmp)) \
 339                 __kfifo_free(__kfifo); \
 340 })
 341 
 342 /**
 343  * kfifo_init - initialize a fifo using a preallocated buffer
 344  * @fifo: the fifo to assign the buffer
 345  * @buffer: the preallocated buffer to be used
 346  * @size: the size of the internal buffer, this have to be a power of 2
 347  *
 348  * This macro initializes a fifo using a preallocated buffer.
 349  *
 350  * The number of elements will be rounded-up to a power of 2.
 351  * Return 0 if no error, otherwise an error code.
 352  */
 353 #define kfifo_init(fifo, buffer, size) \
 354 ({ \
 355         typeof((fifo) + 1) __tmp = (fifo); \
 356         struct __kfifo *__kfifo = &__tmp->kfifo; \
 357         __is_kfifo_ptr(__tmp) ? \
 358         __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
 359         -EINVAL; \
 360 })
 361 
 362 /**
 363  * kfifo_put - put data into the fifo
 364  * @fifo: address of the fifo to be used
 365  * @val: the data to be added
 366  *
 367  * This macro copies the given value into the fifo.
 368  * It returns 0 if the fifo was full. Otherwise it returns the number
 369  * processed elements.
 370  *
 371  * Note that with only one concurrent reader and one concurrent
 372  * writer, you don't need extra locking to use these macro.
 373  */
 374 #define kfifo_put(fifo, val) \
 375 ({ \
 376         typeof((fifo) + 1) __tmp = (fifo); \
 377         typeof(*__tmp->const_type) __val = (val); \
 378         unsigned int __ret; \
 379         size_t __recsize = sizeof(*__tmp->rectype); \
 380         struct __kfifo *__kfifo = &__tmp->kfifo; \
 381         if (__recsize) \
 382                 __ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
 383                         __recsize); \
 384         else { \
 385                 __ret = !kfifo_is_full(__tmp); \
 386                 if (__ret) { \
 387                         (__is_kfifo_ptr(__tmp) ? \
 388                         ((typeof(__tmp->type))__kfifo->data) : \
 389                         (__tmp->buf) \
 390                         )[__kfifo->in & __tmp->kfifo.mask] = \
 391                                 *(typeof(__tmp->type))&__val; \
 392                         smp_wmb(); \
 393                         __kfifo->in++; \
 394                 } \
 395         } \
 396         __ret; \
 397 })
 398 
 399 /**
 400  * kfifo_get - get data from the fifo
 401  * @fifo: address of the fifo to be used
 402  * @val: address where to store the data
 403  *
 404  * This macro reads the data from the fifo.
 405  * It returns 0 if the fifo was empty. Otherwise it returns the number
 406  * processed elements.
 407  *
 408  * Note that with only one concurrent reader and one concurrent
 409  * writer, you don't need extra locking to use these macro.
 410  */
 411 #define kfifo_get(fifo, val) \
 412 __kfifo_uint_must_check_helper( \
 413 ({ \
 414         typeof((fifo) + 1) __tmp = (fifo); \
 415         typeof(__tmp->ptr) __val = (val); \
 416         unsigned int __ret; \
 417         const size_t __recsize = sizeof(*__tmp->rectype); \
 418         struct __kfifo *__kfifo = &__tmp->kfifo; \
 419         if (__recsize) \
 420                 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
 421                         __recsize); \
 422         else { \
 423                 __ret = !kfifo_is_empty(__tmp); \
 424                 if (__ret) { \
 425                         *(typeof(__tmp->type))__val = \
 426                                 (__is_kfifo_ptr(__tmp) ? \
 427                                 ((typeof(__tmp->type))__kfifo->data) : \
 428                                 (__tmp->buf) \
 429                                 )[__kfifo->out & __tmp->kfifo.mask]; \
 430                         smp_wmb(); \
 431                         __kfifo->out++; \
 432                 } \
 433         } \
 434         __ret; \
 435 }) \
 436 )
 437 
 438 /**
 439  * kfifo_peek - get data from the fifo without removing
 440  * @fifo: address of the fifo to be used
 441  * @val: address where to store the data
 442  *
 443  * This reads the data from the fifo without removing it from the fifo.
 444  * It returns 0 if the fifo was empty. Otherwise it returns the number
 445  * processed elements.
 446  *
 447  * Note that with only one concurrent reader and one concurrent
 448  * writer, you don't need extra locking to use these macro.
 449  */
 450 #define kfifo_peek(fifo, val) \
 451 __kfifo_uint_must_check_helper( \
 452 ({ \
 453         typeof((fifo) + 1) __tmp = (fifo); \
 454         typeof(__tmp->ptr) __val = (val); \
 455         unsigned int __ret; \
 456         const size_t __recsize = sizeof(*__tmp->rectype); \
 457         struct __kfifo *__kfifo = &__tmp->kfifo; \
 458         if (__recsize) \
 459                 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
 460                         __recsize); \
 461         else { \
 462                 __ret = !kfifo_is_empty(__tmp); \
 463                 if (__ret) { \
 464                         *(typeof(__tmp->type))__val = \
 465                                 (__is_kfifo_ptr(__tmp) ? \
 466                                 ((typeof(__tmp->type))__kfifo->data) : \
 467                                 (__tmp->buf) \
 468                                 )[__kfifo->out & __tmp->kfifo.mask]; \
 469                         smp_wmb(); \
 470                 } \
 471         } \
 472         __ret; \
 473 }) \
 474 )
 475 
 476 /**
 477  * kfifo_in - put data into the fifo
 478  * @fifo: address of the fifo to be used
 479  * @buf: the data to be added
 480  * @n: number of elements to be added
 481  *
 482  * This macro copies the given buffer into the fifo and returns the
 483  * number of copied elements.
 484  *
 485  * Note that with only one concurrent reader and one concurrent
 486  * writer, you don't need extra locking to use these macro.
 487  */
 488 #define kfifo_in(fifo, buf, n) \
 489 ({ \
 490         typeof((fifo) + 1) __tmp = (fifo); \
 491         typeof(__tmp->ptr_const) __buf = (buf); \
 492         unsigned long __n = (n); \
 493         const size_t __recsize = sizeof(*__tmp->rectype); \
 494         struct __kfifo *__kfifo = &__tmp->kfifo; \
 495         (__recsize) ?\
 496         __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
 497         __kfifo_in(__kfifo, __buf, __n); \
 498 })
 499 
 500 /**
 501  * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
 502  * @fifo: address of the fifo to be used
 503  * @buf: the data to be added
 504  * @n: number of elements to be added
 505  * @lock: pointer to the spinlock to use for locking
 506  *
 507  * This macro copies the given values buffer into the fifo and returns the
 508  * number of copied elements.
 509  */
 510 #define kfifo_in_spinlocked(fifo, buf, n, lock) \
 511 ({ \
 512         unsigned long __flags; \
 513         unsigned int __ret; \
 514         spin_lock_irqsave(lock, __flags); \
 515         __ret = kfifo_in(fifo, buf, n); \
 516         spin_unlock_irqrestore(lock, __flags); \
 517         __ret; \
 518 })
 519 
 520 /* alias for kfifo_in_spinlocked, will be removed in a future release */
 521 #define kfifo_in_locked(fifo, buf, n, lock) \
 522                 kfifo_in_spinlocked(fifo, buf, n, lock)
 523 
 524 /**
 525  * kfifo_out - get data from the fifo
 526  * @fifo: address of the fifo to be used
 527  * @buf: pointer to the storage buffer
 528  * @n: max. number of elements to get
 529  *
 530  * This macro get some data from the fifo and return the numbers of elements
 531  * copied.
 532  *
 533  * Note that with only one concurrent reader and one concurrent
 534  * writer, you don't need extra locking to use these macro.
 535  */
 536 #define kfifo_out(fifo, buf, n) \
 537 __kfifo_uint_must_check_helper( \
 538 ({ \
 539         typeof((fifo) + 1) __tmp = (fifo); \
 540         typeof(__tmp->ptr) __buf = (buf); \
 541         unsigned long __n = (n); \
 542         const size_t __recsize = sizeof(*__tmp->rectype); \
 543         struct __kfifo *__kfifo = &__tmp->kfifo; \
 544         (__recsize) ?\
 545         __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
 546         __kfifo_out(__kfifo, __buf, __n); \
 547 }) \
 548 )
 549 
 550 /**
 551  * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
 552  * @fifo: address of the fifo to be used
 553  * @buf: pointer to the storage buffer
 554  * @n: max. number of elements to get
 555  * @lock: pointer to the spinlock to use for locking
 556  *
 557  * This macro get the data from the fifo and return the numbers of elements
 558  * copied.
 559  */
 560 #define kfifo_out_spinlocked(fifo, buf, n, lock) \
 561 __kfifo_uint_must_check_helper( \
 562 ({ \
 563         unsigned long __flags; \
 564         unsigned int __ret; \
 565         spin_lock_irqsave(lock, __flags); \
 566         __ret = kfifo_out(fifo, buf, n); \
 567         spin_unlock_irqrestore(lock, __flags); \
 568         __ret; \
 569 }) \
 570 )
 571 
 572 /* alias for kfifo_out_spinlocked, will be removed in a future release */
 573 #define kfifo_out_locked(fifo, buf, n, lock) \
 574                 kfifo_out_spinlocked(fifo, buf, n, lock)
 575 
 576 /**
 577  * kfifo_from_user - puts some data from user space into the fifo
 578  * @fifo: address of the fifo to be used
 579  * @from: pointer to the data to be added
 580  * @len: the length of the data to be added
 581  * @copied: pointer to output variable to store the number of copied bytes
 582  *
 583  * This macro copies at most @len bytes from the @from into the
 584  * fifo, depending of the available space and returns -EFAULT/0.
 585  *
 586  * Note that with only one concurrent reader and one concurrent
 587  * writer, you don't need extra locking to use these macro.
 588  */
 589 #define kfifo_from_user(fifo, from, len, copied) \
 590 __kfifo_uint_must_check_helper( \
 591 ({ \
 592         typeof((fifo) + 1) __tmp = (fifo); \
 593         const void __user *__from = (from); \
 594         unsigned int __len = (len); \
 595         unsigned int *__copied = (copied); \
 596         const size_t __recsize = sizeof(*__tmp->rectype); \
 597         struct __kfifo *__kfifo = &__tmp->kfifo; \
 598         (__recsize) ? \
 599         __kfifo_from_user_r(__kfifo, __from, __len,  __copied, __recsize) : \
 600         __kfifo_from_user(__kfifo, __from, __len, __copied); \
 601 }) \
 602 )
 603 
 604 /**
 605  * kfifo_to_user - copies data from the fifo into user space
 606  * @fifo: address of the fifo to be used
 607  * @to: where the data must be copied
 608  * @len: the size of the destination buffer
 609  * @copied: pointer to output variable to store the number of copied bytes
 610  *
 611  * This macro copies at most @len bytes from the fifo into the
 612  * @to buffer and returns -EFAULT/0.
 613  *
 614  * Note that with only one concurrent reader and one concurrent
 615  * writer, you don't need extra locking to use these macro.
 616  */
 617 #define kfifo_to_user(fifo, to, len, copied) \
 618 __kfifo_uint_must_check_helper( \
 619 ({ \
 620         typeof((fifo) + 1) __tmp = (fifo); \
 621         void __user *__to = (to); \
 622         unsigned int __len = (len); \
 623         unsigned int *__copied = (copied); \
 624         const size_t __recsize = sizeof(*__tmp->rectype); \
 625         struct __kfifo *__kfifo = &__tmp->kfifo; \
 626         (__recsize) ? \
 627         __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
 628         __kfifo_to_user(__kfifo, __to, __len, __copied); \
 629 }) \
 630 )
 631 
 632 /**
 633  * kfifo_dma_in_prepare - setup a scatterlist for DMA input
 634  * @fifo: address of the fifo to be used
 635  * @sgl: pointer to the scatterlist array
 636  * @nents: number of entries in the scatterlist array
 637  * @len: number of elements to transfer
 638  *
 639  * This macro fills a scatterlist for DMA input.
 640  * It returns the number entries in the scatterlist array.
 641  *
 642  * Note that with only one concurrent reader and one concurrent
 643  * writer, you don't need extra locking to use these macros.
 644  */
 645 #define kfifo_dma_in_prepare(fifo, sgl, nents, len) \
 646 ({ \
 647         typeof((fifo) + 1) __tmp = (fifo); \
 648         struct scatterlist *__sgl = (sgl); \
 649         int __nents = (nents); \
 650         unsigned int __len = (len); \
 651         const size_t __recsize = sizeof(*__tmp->rectype); \
 652         struct __kfifo *__kfifo = &__tmp->kfifo; \
 653         (__recsize) ? \
 654         __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
 655         __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
 656 })
 657 
 658 /**
 659  * kfifo_dma_in_finish - finish a DMA IN operation
 660  * @fifo: address of the fifo to be used
 661  * @len: number of bytes to received
 662  *
 663  * This macro finish a DMA IN operation. The in counter will be updated by
 664  * the len parameter. No error checking will be done.
 665  *
 666  * Note that with only one concurrent reader and one concurrent
 667  * writer, you don't need extra locking to use these macros.
 668  */
 669 #define kfifo_dma_in_finish(fifo, len) \
 670 (void)({ \
 671         typeof((fifo) + 1) __tmp = (fifo); \
 672         unsigned int __len = (len); \
 673         const size_t __recsize = sizeof(*__tmp->rectype); \
 674         struct __kfifo *__kfifo = &__tmp->kfifo; \
 675         if (__recsize) \
 676                 __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
 677         else \
 678                 __kfifo->in += __len / sizeof(*__tmp->type); \
 679 })
 680 
 681 /**
 682  * kfifo_dma_out_prepare - setup a scatterlist for DMA output
 683  * @fifo: address of the fifo to be used
 684  * @sgl: pointer to the scatterlist array
 685  * @nents: number of entries in the scatterlist array
 686  * @len: number of elements to transfer
 687  *
 688  * This macro fills a scatterlist for DMA output which at most @len bytes
 689  * to transfer.
 690  * It returns the number entries in the scatterlist array.
 691  * A zero means there is no space available and the scatterlist is not filled.
 692  *
 693  * Note that with only one concurrent reader and one concurrent
 694  * writer, you don't need extra locking to use these macros.
 695  */
 696 #define kfifo_dma_out_prepare(fifo, sgl, nents, len) \
 697 ({ \
 698         typeof((fifo) + 1) __tmp = (fifo);  \
 699         struct scatterlist *__sgl = (sgl); \
 700         int __nents = (nents); \
 701         unsigned int __len = (len); \
 702         const size_t __recsize = sizeof(*__tmp->rectype); \
 703         struct __kfifo *__kfifo = &__tmp->kfifo; \
 704         (__recsize) ? \
 705         __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
 706         __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
 707 })
 708 
 709 /**
 710  * kfifo_dma_out_finish - finish a DMA OUT operation
 711  * @fifo: address of the fifo to be used
 712  * @len: number of bytes transferred
 713  *
 714  * This macro finish a DMA OUT operation. The out counter will be updated by
 715  * the len parameter. No error checking will be done.
 716  *
 717  * Note that with only one concurrent reader and one concurrent
 718  * writer, you don't need extra locking to use these macros.
 719  */
 720 #define kfifo_dma_out_finish(fifo, len) \
 721 (void)({ \
 722         typeof((fifo) + 1) __tmp = (fifo); \
 723         unsigned int __len = (len); \
 724         const size_t __recsize = sizeof(*__tmp->rectype); \
 725         struct __kfifo *__kfifo = &__tmp->kfifo; \
 726         if (__recsize) \
 727                 __kfifo_dma_out_finish_r(__kfifo, __recsize); \
 728         else \
 729                 __kfifo->out += __len / sizeof(*__tmp->type); \
 730 })
 731 
 732 /**
 733  * kfifo_out_peek - gets some data from the fifo
 734  * @fifo: address of the fifo to be used
 735  * @buf: pointer to the storage buffer
 736  * @n: max. number of elements to get
 737  *
 738  * This macro get the data from the fifo and return the numbers of elements
 739  * copied. The data is not removed from the fifo.
 740  *
 741  * Note that with only one concurrent reader and one concurrent
 742  * writer, you don't need extra locking to use these macro.
 743  */
 744 #define kfifo_out_peek(fifo, buf, n) \
 745 __kfifo_uint_must_check_helper( \
 746 ({ \
 747         typeof((fifo) + 1) __tmp = (fifo); \
 748         typeof(__tmp->ptr) __buf = (buf); \
 749         unsigned long __n = (n); \
 750         const size_t __recsize = sizeof(*__tmp->rectype); \
 751         struct __kfifo *__kfifo = &__tmp->kfifo; \
 752         (__recsize) ? \
 753         __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
 754         __kfifo_out_peek(__kfifo, __buf, __n); \
 755 }) \
 756 )
 757 
 758 extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
 759         size_t esize, gfp_t gfp_mask);
 760 
 761 extern void __kfifo_free(struct __kfifo *fifo);
 762 
 763 extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
 764         unsigned int size, size_t esize);
 765 
 766 extern unsigned int __kfifo_in(struct __kfifo *fifo,
 767         const void *buf, unsigned int len);
 768 
 769 extern unsigned int __kfifo_out(struct __kfifo *fifo,
 770         void *buf, unsigned int len);
 771 
 772 extern int __kfifo_from_user(struct __kfifo *fifo,
 773         const void __user *from, unsigned long len, unsigned int *copied);
 774 
 775 extern int __kfifo_to_user(struct __kfifo *fifo,
 776         void __user *to, unsigned long len, unsigned int *copied);
 777 
 778 extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
 779         struct scatterlist *sgl, int nents, unsigned int len);
 780 
 781 extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
 782         struct scatterlist *sgl, int nents, unsigned int len);
 783 
 784 extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
 785         void *buf, unsigned int len);
 786 
 787 extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
 788         const void *buf, unsigned int len, size_t recsize);
 789 
 790 extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
 791         void *buf, unsigned int len, size_t recsize);
 792 
 793 extern int __kfifo_from_user_r(struct __kfifo *fifo,
 794         const void __user *from, unsigned long len, unsigned int *copied,
 795         size_t recsize);
 796 
 797 extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
 798         unsigned long len, unsigned int *copied, size_t recsize);
 799 
 800 extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
 801         struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
 802 
 803 extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
 804         unsigned int len, size_t recsize);
 805 
 806 extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
 807         struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
 808 
 809 extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
 810 
 811 extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
 812 
 813 extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
 814 
 815 extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
 816         void *buf, unsigned int len, size_t recsize);
 817 
 818 extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
 819 
 820 #endif