1 /*
2  *	Definitions for the 'struct sk_buff' memory handlers.
3  *
4  *	Authors:
5  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
6  *		Florian La Roche, <rzsfl@rz.uni-sb.de>
7  *
8  *	This program is free software; you can redistribute it and/or
9  *	modify it under the terms of the GNU General Public License
10  *	as published by the Free Software Foundation; either version
11  *	2 of the License, or (at your option) any later version.
12  */
13 
14 #ifndef _LINUX_SKBUFF_H
15 #define _LINUX_SKBUFF_H
16 
17 #include <linux/kernel.h>
18 #include <linux/kmemcheck.h>
19 #include <linux/compiler.h>
20 #include <linux/time.h>
21 #include <linux/bug.h>
22 #include <linux/cache.h>
23 #include <linux/rbtree.h>
24 #include <linux/socket.h>
25 
26 #include <linux/atomic.h>
27 #include <asm/types.h>
28 #include <linux/spinlock.h>
29 #include <linux/net.h>
30 #include <linux/textsearch.h>
31 #include <net/checksum.h>
32 #include <linux/rcupdate.h>
33 #include <linux/hrtimer.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/netdev_features.h>
36 #include <linux/sched.h>
37 #include <net/flow_dissector.h>
38 #include <linux/splice.h>
39 #include <linux/in6.h>
40 #include <net/flow.h>
41 
42 /* A. Checksumming of received packets by device.
43  *
44  * CHECKSUM_NONE:
45  *
46  *   Device failed to checksum this packet e.g. due to lack of capabilities.
47  *   The packet contains full (though not verified) checksum in packet but
48  *   not in skb->csum. Thus, skb->csum is undefined in this case.
49  *
50  * CHECKSUM_UNNECESSARY:
51  *
52  *   The hardware you're dealing with doesn't calculate the full checksum
53  *   (as in CHECKSUM_COMPLETE), but it does parse headers and verify checksums
54  *   for specific protocols. For such packets it will set CHECKSUM_UNNECESSARY
55  *   if their checksums are okay. skb->csum is still undefined in this case
56  *   though. It is a bad option, but, unfortunately, nowadays most vendors do
57  *   this. Apparently with the secret goal to sell you new devices, when you
58  *   will add new protocol to your host, f.e. IPv6 8)
59  *
60  *   CHECKSUM_UNNECESSARY is applicable to following protocols:
61  *     TCP: IPv6 and IPv4.
62  *     UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
63  *       zero UDP checksum for either IPv4 or IPv6, the networking stack
64  *       may perform further validation in this case.
65  *     GRE: only if the checksum is present in the header.
66  *     SCTP: indicates the CRC in SCTP header has been validated.
67  *
68  *   skb->csum_level indicates the number of consecutive checksums found in
69  *   the packet minus one that have been verified as CHECKSUM_UNNECESSARY.
70  *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
71  *   and a device is able to verify the checksums for UDP (possibly zero),
72  *   GRE (checksum flag is set), and TCP-- skb->csum_level would be set to
73  *   two. If the device were only able to verify the UDP checksum and not
74  *   GRE, either because it doesn't support GRE checksum of because GRE
75  *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
76  *   not considered in this case).
77  *
78  * CHECKSUM_COMPLETE:
79  *
80  *   This is the most generic way. The device supplied checksum of the _whole_
81  *   packet as seen by netif_rx() and fills out in skb->csum. Meaning, the
82  *   hardware doesn't need to parse L3/L4 headers to implement this.
83  *
84  *   Note: Even if device supports only some protocols, but is able to produce
85  *   skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
86  *
87  * CHECKSUM_PARTIAL:
88  *
89  *   A checksum is set up to be offloaded to a device as described in the
90  *   output description for CHECKSUM_PARTIAL. This may occur on a packet
91  *   received directly from another Linux OS, e.g., a virtualized Linux kernel
92  *   on the same host, or it may be set in the input path in GRO or remote
93  *   checksum offload. For the purposes of checksum verification, the checksum
94  *   referred to by skb->csum_start + skb->csum_offset and any preceding
95  *   checksums in the packet are considered verified. Any checksums in the
96  *   packet that are after the checksum being offloaded are not considered to
97  *   be verified.
98  *
99  * B. Checksumming on output.
100  *
101  * CHECKSUM_NONE:
102  *
103  *   The skb was already checksummed by the protocol, or a checksum is not
104  *   required.
105  *
106  * CHECKSUM_PARTIAL:
107  *
108  *   The device is required to checksum the packet as seen by hard_start_xmit()
109  *   from skb->csum_start up to the end, and to record/write the checksum at
110  *   offset skb->csum_start + skb->csum_offset.
111  *
112  *   The device must show its capabilities in dev->features, set up at device
113  *   setup time, e.g. netdev_features.h:
114  *
115  *	NETIF_F_HW_CSUM	- It's a clever device, it's able to checksum everything.
116  *	NETIF_F_IP_CSUM - Device is dumb, it's able to checksum only TCP/UDP over
117  *			  IPv4. Sigh. Vendors like this way for an unknown reason.
118  *			  Though, see comment above about CHECKSUM_UNNECESSARY. 8)
119  *	NETIF_F_IPV6_CSUM - About as dumb as the last one but does IPv6 instead.
120  *	NETIF_F_...     - Well, you get the picture.
121  *
122  * CHECKSUM_UNNECESSARY:
123  *
124  *   Normally, the device will do per protocol specific checksumming. Protocol
125  *   implementations that do not want the NIC to perform the checksum
126  *   calculation should use this flag in their outgoing skbs.
127  *
128  *	NETIF_F_FCOE_CRC - This indicates that the device can do FCoE FC CRC
129  *			   offload. Correspondingly, the FCoE protocol driver
130  *			   stack should use CHECKSUM_UNNECESSARY.
131  *
132  * Any questions? No questions, good.		--ANK
133  */
134 
135 /* Don't change this without changing skb_csum_unnecessary! */
136 #define CHECKSUM_NONE		0
137 #define CHECKSUM_UNNECESSARY	1
138 #define CHECKSUM_COMPLETE	2
139 #define CHECKSUM_PARTIAL	3
140 
141 /* Maximum value in skb->csum_level */
142 #define SKB_MAX_CSUM_LEVEL	3
143 
144 #define SKB_DATA_ALIGN(X)	ALIGN(X, SMP_CACHE_BYTES)
145 #define SKB_WITH_OVERHEAD(X)	\
146 	((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
147 #define SKB_MAX_ORDER(X, ORDER) \
148 	SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
149 #define SKB_MAX_HEAD(X)		(SKB_MAX_ORDER((X), 0))
150 #define SKB_MAX_ALLOC		(SKB_MAX_ORDER(0, 2))
151 
152 /* return minimum truesize of one skb containing X bytes of data */
153 #define SKB_TRUESIZE(X) ((X) +						\
154 			 SKB_DATA_ALIGN(sizeof(struct sk_buff)) +	\
155 			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
156 
157 struct net_device;
158 struct scatterlist;
159 struct pipe_inode_info;
160 struct iov_iter;
161 struct napi_struct;
162 
163 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
164 struct nf_conntrack {
165 	atomic_t use;
166 };
167 #endif
168 
169 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
170 struct nf_bridge_info {
171 	atomic_t		use;
172 	enum {
173 		BRNF_PROTO_UNCHANGED,
174 		BRNF_PROTO_8021Q,
175 		BRNF_PROTO_PPPOE
176 	} orig_proto:8;
177 	u8			pkt_otherhost:1;
178 	u8			in_prerouting:1;
179 	u8			bridged_dnat:1;
180 	__u16			frag_max_size;
181 	struct net_device	*physindev;
182 
183 	/* always valid & non-NULL from FORWARD on, for physdev match */
184 	struct net_device	*physoutdev;
185 	union {
186 		/* prerouting: detect dnat in orig/reply direction */
187 		__be32          ipv4_daddr;
188 		struct in6_addr ipv6_daddr;
189 
190 		/* after prerouting + nat detected: store original source
191 		 * mac since neigh resolution overwrites it, only used while
192 		 * skb is out in neigh layer.
193 		 */
194 		char neigh_header[8];
195 	};
196 };
197 #endif
198 
199 struct sk_buff_head {
200 	/* These two members must be first. */
201 	struct sk_buff	*next;
202 	struct sk_buff	*prev;
203 
204 	__u32		qlen;
205 	spinlock_t	lock;
206 };
207 
208 struct sk_buff;
209 
210 /* To allow 64K frame to be packed as single skb without frag_list we
211  * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
212  * buffers which do not start on a page boundary.
213  *
214  * Since GRO uses frags we allocate at least 16 regardless of page
215  * size.
216  */
217 #if (65536/PAGE_SIZE + 1) < 16
218 #define MAX_SKB_FRAGS 16UL
219 #else
220 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
221 #endif
222 extern int sysctl_max_skb_frags;
223 
224 typedef struct skb_frag_struct skb_frag_t;
225 
226 struct skb_frag_struct {
227 	struct {
228 		struct page *p;
229 	} page;
230 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
231 	__u32 page_offset;
232 	__u32 size;
233 #else
234 	__u16 page_offset;
235 	__u16 size;
236 #endif
237 };
238 
skb_frag_size(const skb_frag_t * frag)239 static inline unsigned int skb_frag_size(const skb_frag_t *frag)
240 {
241 	return frag->size;
242 }
243 
skb_frag_size_set(skb_frag_t * frag,unsigned int size)244 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
245 {
246 	frag->size = size;
247 }
248 
skb_frag_size_add(skb_frag_t * frag,int delta)249 static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
250 {
251 	frag->size += delta;
252 }
253 
skb_frag_size_sub(skb_frag_t * frag,int delta)254 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
255 {
256 	frag->size -= delta;
257 }
258 
259 #define HAVE_HW_TIME_STAMP
260 
261 /**
262  * struct skb_shared_hwtstamps - hardware time stamps
263  * @hwtstamp:	hardware time stamp transformed into duration
264  *		since arbitrary point in time
265  *
266  * Software time stamps generated by ktime_get_real() are stored in
267  * skb->tstamp.
268  *
269  * hwtstamps can only be compared against other hwtstamps from
270  * the same device.
271  *
272  * This structure is attached to packets as part of the
273  * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
274  */
275 struct skb_shared_hwtstamps {
276 	ktime_t	hwtstamp;
277 };
278 
279 /* Definitions for tx_flags in struct skb_shared_info */
280 enum {
281 	/* generate hardware time stamp */
282 	SKBTX_HW_TSTAMP = 1 << 0,
283 
284 	/* generate software time stamp when queueing packet to NIC */
285 	SKBTX_SW_TSTAMP = 1 << 1,
286 
287 	/* device driver is going to provide hardware time stamp */
288 	SKBTX_IN_PROGRESS = 1 << 2,
289 
290 	/* device driver supports TX zero-copy buffers */
291 	SKBTX_DEV_ZEROCOPY = 1 << 3,
292 
293 	/* generate wifi status information (where possible) */
294 	SKBTX_WIFI_STATUS = 1 << 4,
295 
296 	/* This indicates at least one fragment might be overwritten
297 	 * (as in vmsplice(), sendfile() ...)
298 	 * If we need to compute a TX checksum, we'll need to copy
299 	 * all frags to avoid possible bad checksum
300 	 */
301 	SKBTX_SHARED_FRAG = 1 << 5,
302 
303 	/* generate software time stamp when entering packet scheduling */
304 	SKBTX_SCHED_TSTAMP = 1 << 6,
305 
306 	/* generate software timestamp on peer data acknowledgment */
307 	SKBTX_ACK_TSTAMP = 1 << 7,
308 };
309 
310 #define SKBTX_ANY_SW_TSTAMP	(SKBTX_SW_TSTAMP    | \
311 				 SKBTX_SCHED_TSTAMP | \
312 				 SKBTX_ACK_TSTAMP)
313 #define SKBTX_ANY_TSTAMP	(SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP)
314 
315 /*
316  * The callback notifies userspace to release buffers when skb DMA is done in
317  * lower device, the skb last reference should be 0 when calling this.
318  * The zerocopy_success argument is true if zero copy transmit occurred,
319  * false on data copy or out of memory error caused by data copy attempt.
320  * The ctx field is used to track device context.
321  * The desc field is used to track userspace buffer index.
322  */
323 struct ubuf_info {
324 	void (*callback)(struct ubuf_info *, bool zerocopy_success);
325 	void *ctx;
326 	unsigned long desc;
327 };
328 
329 /* This data is invariant across clones and lives at
330  * the end of the header data, ie. at skb->end.
331  */
332 struct skb_shared_info {
333 	unsigned char	nr_frags;
334 	__u8		tx_flags;
335 	unsigned short	gso_size;
336 	/* Warning: this field is not always filled in (UFO)! */
337 	unsigned short	gso_segs;
338 	unsigned short  gso_type;
339 	struct sk_buff	*frag_list;
340 	struct skb_shared_hwtstamps hwtstamps;
341 	u32		tskey;
342 	__be32          ip6_frag_id;
343 
344 	/*
345 	 * Warning : all fields before dataref are cleared in __alloc_skb()
346 	 */
347 	atomic_t	dataref;
348 
349 	/* Intermediate layers must ensure that destructor_arg
350 	 * remains valid until skb destructor */
351 	void *		destructor_arg;
352 
353 	/* must be last field, see pskb_expand_head() */
354 	skb_frag_t	frags[MAX_SKB_FRAGS];
355 };
356 
357 /* We divide dataref into two halves.  The higher 16 bits hold references
358  * to the payload part of skb->data.  The lower 16 bits hold references to
359  * the entire skb->data.  A clone of a headerless skb holds the length of
360  * the header in skb->hdr_len.
361  *
362  * All users must obey the rule that the skb->data reference count must be
363  * greater than or equal to the payload reference count.
364  *
365  * Holding a reference to the payload part means that the user does not
366  * care about modifications to the header part of skb->data.
367  */
368 #define SKB_DATAREF_SHIFT 16
369 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
370 
371 
372 enum {
373 	SKB_FCLONE_UNAVAILABLE,	/* skb has no fclone (from head_cache) */
374 	SKB_FCLONE_ORIG,	/* orig skb (from fclone_cache) */
375 	SKB_FCLONE_CLONE,	/* companion fclone skb (from fclone_cache) */
376 };
377 
378 enum {
379 	SKB_GSO_TCPV4 = 1 << 0,
380 	SKB_GSO_UDP = 1 << 1,
381 
382 	/* This indicates the skb is from an untrusted source. */
383 	SKB_GSO_DODGY = 1 << 2,
384 
385 	/* This indicates the tcp segment has CWR set. */
386 	SKB_GSO_TCP_ECN = 1 << 3,
387 
388 	SKB_GSO_TCPV6 = 1 << 4,
389 
390 	SKB_GSO_FCOE = 1 << 5,
391 
392 	SKB_GSO_GRE = 1 << 6,
393 
394 	SKB_GSO_GRE_CSUM = 1 << 7,
395 
396 	SKB_GSO_IPIP = 1 << 8,
397 
398 	SKB_GSO_SIT = 1 << 9,
399 
400 	SKB_GSO_UDP_TUNNEL = 1 << 10,
401 
402 	SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
403 
404 	SKB_GSO_TUNNEL_REMCSUM = 1 << 12,
405 };
406 
407 #if BITS_PER_LONG > 32
408 #define NET_SKBUFF_DATA_USES_OFFSET 1
409 #endif
410 
411 #ifdef NET_SKBUFF_DATA_USES_OFFSET
412 typedef unsigned int sk_buff_data_t;
413 #else
414 typedef unsigned char *sk_buff_data_t;
415 #endif
416 
417 /**
418  * struct skb_mstamp - multi resolution time stamps
419  * @stamp_us: timestamp in us resolution
420  * @stamp_jiffies: timestamp in jiffies
421  */
422 struct skb_mstamp {
423 	union {
424 		u64		v64;
425 		struct {
426 			u32	stamp_us;
427 			u32	stamp_jiffies;
428 		};
429 	};
430 };
431 
432 /**
433  * skb_mstamp_get - get current timestamp
434  * @cl: place to store timestamps
435  */
skb_mstamp_get(struct skb_mstamp * cl)436 static inline void skb_mstamp_get(struct skb_mstamp *cl)
437 {
438 	u64 val = local_clock();
439 
440 	do_div(val, NSEC_PER_USEC);
441 	cl->stamp_us = (u32)val;
442 	cl->stamp_jiffies = (u32)jiffies;
443 }
444 
445 /**
446  * skb_mstamp_delta - compute the difference in usec between two skb_mstamp
447  * @t1: pointer to newest sample
448  * @t0: pointer to oldest sample
449  */
skb_mstamp_us_delta(const struct skb_mstamp * t1,const struct skb_mstamp * t0)450 static inline u32 skb_mstamp_us_delta(const struct skb_mstamp *t1,
451 				      const struct skb_mstamp *t0)
452 {
453 	s32 delta_us = t1->stamp_us - t0->stamp_us;
454 	u32 delta_jiffies = t1->stamp_jiffies - t0->stamp_jiffies;
455 
456 	/* If delta_us is negative, this might be because interval is too big,
457 	 * or local_clock() drift is too big : fallback using jiffies.
458 	 */
459 	if (delta_us <= 0 ||
460 	    delta_jiffies >= (INT_MAX / (USEC_PER_SEC / HZ)))
461 
462 		delta_us = jiffies_to_usecs(delta_jiffies);
463 
464 	return delta_us;
465 }
466 
skb_mstamp_after(const struct skb_mstamp * t1,const struct skb_mstamp * t0)467 static inline bool skb_mstamp_after(const struct skb_mstamp *t1,
468 				    const struct skb_mstamp *t0)
469 {
470 	s32 diff = t1->stamp_jiffies - t0->stamp_jiffies;
471 
472 	if (!diff)
473 		diff = t1->stamp_us - t0->stamp_us;
474 	return diff > 0;
475 }
476 
477 /**
478  *	struct sk_buff - socket buffer
479  *	@next: Next buffer in list
480  *	@prev: Previous buffer in list
481  *	@tstamp: Time we arrived/left
482  *	@rbnode: RB tree node, alternative to next/prev for netem/tcp
483  *	@sk: Socket we are owned by
484  *	@dev: Device we arrived on/are leaving by
485  *	@cb: Control buffer. Free for use by every layer. Put private vars here
486  *	@_skb_refdst: destination entry (with norefcount bit)
487  *	@sp: the security path, used for xfrm
488  *	@len: Length of actual data
489  *	@data_len: Data length
490  *	@mac_len: Length of link layer header
491  *	@hdr_len: writable header length of cloned skb
492  *	@csum: Checksum (must include start/offset pair)
493  *	@csum_start: Offset from skb->head where checksumming should start
494  *	@csum_offset: Offset from csum_start where checksum should be stored
495  *	@priority: Packet queueing priority
496  *	@ignore_df: allow local fragmentation
497  *	@cloned: Head may be cloned (check refcnt to be sure)
498  *	@ip_summed: Driver fed us an IP checksum
499  *	@nohdr: Payload reference only, must not modify header
500  *	@nfctinfo: Relationship of this skb to the connection
501  *	@pkt_type: Packet class
502  *	@fclone: skbuff clone status
503  *	@ipvs_property: skbuff is owned by ipvs
504  *	@peeked: this packet has been seen already, so stats have been
505  *		done for it, don't do them again
506  *	@nf_trace: netfilter packet trace flag
507  *	@protocol: Packet protocol from driver
508  *	@destructor: Destruct function
509  *	@nfct: Associated connection, if any
510  *	@nf_bridge: Saved data about a bridged frame - see br_netfilter.c
511  *	@skb_iif: ifindex of device we arrived on
512  *	@tc_index: Traffic control index
513  *	@tc_verd: traffic control verdict
514  *	@hash: the packet hash
515  *	@queue_mapping: Queue mapping for multiqueue devices
516  *	@xmit_more: More SKBs are pending for this queue
517  *	@ndisc_nodetype: router type (from link layer)
518  *	@ooo_okay: allow the mapping of a socket to a queue to be changed
519  *	@l4_hash: indicate hash is a canonical 4-tuple hash over transport
520  *		ports.
521  *	@sw_hash: indicates hash was computed in software stack
522  *	@wifi_acked_valid: wifi_acked was set
523  *	@wifi_acked: whether frame was acked on wifi or not
524  *	@no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
525   *	@napi_id: id of the NAPI struct this skb came from
526  *	@secmark: security marking
527  *	@offload_fwd_mark: fwding offload mark
528  *	@mark: Generic packet mark
529  *	@vlan_proto: vlan encapsulation protocol
530  *	@vlan_tci: vlan tag control information
531  *	@inner_protocol: Protocol (encapsulation)
532  *	@inner_transport_header: Inner transport layer header (encapsulation)
533  *	@inner_network_header: Network layer header (encapsulation)
534  *	@inner_mac_header: Link layer header (encapsulation)
535  *	@transport_header: Transport layer header
536  *	@network_header: Network layer header
537  *	@mac_header: Link layer header
538  *	@tail: Tail pointer
539  *	@end: End pointer
540  *	@head: Head of buffer
541  *	@data: Data head pointer
542  *	@truesize: Buffer size
543  *	@users: User count - see {datagram,tcp}.c
544  */
545 
546 struct sk_buff {
547 	union {
548 		struct {
549 			/* These two members must be first. */
550 			struct sk_buff		*next;
551 			struct sk_buff		*prev;
552 
553 			union {
554 				ktime_t		tstamp;
555 				struct skb_mstamp skb_mstamp;
556 			};
557 		};
558 		struct rb_node	rbnode; /* used in netem & tcp stack */
559 	};
560 	struct sock		*sk;
561 	struct net_device	*dev;
562 
563 	/*
564 	 * This is the control buffer. It is free to use for every
565 	 * layer. Please put your private variables there. If you
566 	 * want to keep them across layers you have to do a skb_clone()
567 	 * first. This is owned by whoever has the skb queued ATM.
568 	 */
569 	char			cb[48] __aligned(8);
570 
571 	unsigned long		_skb_refdst;
572 	void			(*destructor)(struct sk_buff *skb);
573 #ifdef CONFIG_XFRM
574 	struct	sec_path	*sp;
575 #endif
576 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
577 	struct nf_conntrack	*nfct;
578 #endif
579 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
580 	struct nf_bridge_info	*nf_bridge;
581 #endif
582 	unsigned int		len,
583 				data_len;
584 	__u16			mac_len,
585 				hdr_len;
586 
587 	/* Following fields are _not_ copied in __copy_skb_header()
588 	 * Note that queue_mapping is here mostly to fill a hole.
589 	 */
590 	kmemcheck_bitfield_begin(flags1);
591 	__u16			queue_mapping;
592 	__u8			cloned:1,
593 				nohdr:1,
594 				fclone:2,
595 				peeked:1,
596 				head_frag:1,
597 				xmit_more:1;
598 	/* one bit hole */
599 	kmemcheck_bitfield_end(flags1);
600 
601 	/* fields enclosed in headers_start/headers_end are copied
602 	 * using a single memcpy() in __copy_skb_header()
603 	 */
604 	/* private: */
605 	__u32			headers_start[0];
606 	/* public: */
607 
608 /* if you move pkt_type around you also must adapt those constants */
609 #ifdef __BIG_ENDIAN_BITFIELD
610 #define PKT_TYPE_MAX	(7 << 5)
611 #else
612 #define PKT_TYPE_MAX	7
613 #endif
614 #define PKT_TYPE_OFFSET()	offsetof(struct sk_buff, __pkt_type_offset)
615 
616 	__u8			__pkt_type_offset[0];
617 	__u8			pkt_type:3;
618 	__u8			pfmemalloc:1;
619 	__u8			ignore_df:1;
620 	__u8			nfctinfo:3;
621 
622 	__u8			nf_trace:1;
623 	__u8			ip_summed:2;
624 	__u8			ooo_okay:1;
625 	__u8			l4_hash:1;
626 	__u8			sw_hash:1;
627 	__u8			wifi_acked_valid:1;
628 	__u8			wifi_acked:1;
629 
630 	__u8			no_fcs:1;
631 	/* Indicates the inner headers are valid in the skbuff. */
632 	__u8			encapsulation:1;
633 	__u8			encap_hdr_csum:1;
634 	__u8			csum_valid:1;
635 	__u8			csum_complete_sw:1;
636 	__u8			csum_level:2;
637 	__u8			csum_bad:1;
638 
639 #ifdef CONFIG_IPV6_NDISC_NODETYPE
640 	__u8			ndisc_nodetype:2;
641 #endif
642 	__u8			ipvs_property:1;
643 	__u8			inner_protocol_type:1;
644 	__u8			remcsum_offload:1;
645 	/* 3 or 5 bit hole */
646 
647 #ifdef CONFIG_NET_SCHED
648 	__u16			tc_index;	/* traffic control index */
649 #ifdef CONFIG_NET_CLS_ACT
650 	__u16			tc_verd;	/* traffic control verdict */
651 #endif
652 #endif
653 
654 	union {
655 		__wsum		csum;
656 		struct {
657 			__u16	csum_start;
658 			__u16	csum_offset;
659 		};
660 	};
661 	__u32			priority;
662 	int			skb_iif;
663 	__u32			hash;
664 	__be16			vlan_proto;
665 	__u16			vlan_tci;
666 #if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
667 	union {
668 		unsigned int	napi_id;
669 		unsigned int	sender_cpu;
670 	};
671 #endif
672 	union {
673 #ifdef CONFIG_NETWORK_SECMARK
674 		__u32		secmark;
675 #endif
676 #ifdef CONFIG_NET_SWITCHDEV
677 		__u32		offload_fwd_mark;
678 #endif
679 	};
680 
681 	union {
682 		__u32		mark;
683 		__u32		reserved_tailroom;
684 	};
685 
686 	union {
687 		__be16		inner_protocol;
688 		__u8		inner_ipproto;
689 	};
690 
691 	__u16			inner_transport_header;
692 	__u16			inner_network_header;
693 	__u16			inner_mac_header;
694 
695 	__be16			protocol;
696 	__u16			transport_header;
697 	__u16			network_header;
698 	__u16			mac_header;
699 
700 	/* private: */
701 	__u32			headers_end[0];
702 	/* public: */
703 
704 	/* These elements must be at the end, see alloc_skb() for details.  */
705 	sk_buff_data_t		tail;
706 	sk_buff_data_t		end;
707 	unsigned char		*head,
708 				*data;
709 	unsigned int		truesize;
710 	atomic_t		users;
711 };
712 
713 #ifdef __KERNEL__
714 /*
715  *	Handling routines are only of interest to the kernel
716  */
717 #include <linux/slab.h>
718 
719 
720 #define SKB_ALLOC_FCLONE	0x01
721 #define SKB_ALLOC_RX		0x02
722 #define SKB_ALLOC_NAPI		0x04
723 
724 /* Returns true if the skb was allocated from PFMEMALLOC reserves */
skb_pfmemalloc(const struct sk_buff * skb)725 static inline bool skb_pfmemalloc(const struct sk_buff *skb)
726 {
727 	return unlikely(skb->pfmemalloc);
728 }
729 
730 /*
731  * skb might have a dst pointer attached, refcounted or not.
732  * _skb_refdst low order bit is set if refcount was _not_ taken
733  */
734 #define SKB_DST_NOREF	1UL
735 #define SKB_DST_PTRMASK	~(SKB_DST_NOREF)
736 
737 /**
738  * skb_dst - returns skb dst_entry
739  * @skb: buffer
740  *
741  * Returns skb dst_entry, regardless of reference taken or not.
742  */
skb_dst(const struct sk_buff * skb)743 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
744 {
745 	/* If refdst was not refcounted, check we still are in a
746 	 * rcu_read_lock section
747 	 */
748 	WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
749 		!rcu_read_lock_held() &&
750 		!rcu_read_lock_bh_held());
751 	return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
752 }
753 
754 /**
755  * skb_dst_set - sets skb dst
756  * @skb: buffer
757  * @dst: dst entry
758  *
759  * Sets skb dst, assuming a reference was taken on dst and should
760  * be released by skb_dst_drop()
761  */
skb_dst_set(struct sk_buff * skb,struct dst_entry * dst)762 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
763 {
764 	skb->_skb_refdst = (unsigned long)dst;
765 }
766 
767 /**
768  * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
769  * @skb: buffer
770  * @dst: dst entry
771  *
772  * Sets skb dst, assuming a reference was not taken on dst.
773  * If dst entry is cached, we do not take reference and dst_release
774  * will be avoided by refdst_drop. If dst entry is not cached, we take
775  * reference, so that last dst_release can destroy the dst immediately.
776  */
skb_dst_set_noref(struct sk_buff * skb,struct dst_entry * dst)777 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
778 {
779 	WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
780 	skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
781 }
782 
783 /**
784  * skb_dst_is_noref - Test if skb dst isn't refcounted
785  * @skb: buffer
786  */
skb_dst_is_noref(const struct sk_buff * skb)787 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
788 {
789 	return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
790 }
791 
skb_rtable(const struct sk_buff * skb)792 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
793 {
794 	return (struct rtable *)skb_dst(skb);
795 }
796 
797 void kfree_skb(struct sk_buff *skb);
798 void kfree_skb_list(struct sk_buff *segs);
799 void skb_tx_error(struct sk_buff *skb);
800 void consume_skb(struct sk_buff *skb);
801 void  __kfree_skb(struct sk_buff *skb);
802 extern struct kmem_cache *skbuff_head_cache;
803 
804 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
805 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
806 		      bool *fragstolen, int *delta_truesize);
807 
808 struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
809 			    int node);
810 struct sk_buff *__build_skb(void *data, unsigned int frag_size);
811 struct sk_buff *build_skb(void *data, unsigned int frag_size);
alloc_skb(unsigned int size,gfp_t priority)812 static inline struct sk_buff *alloc_skb(unsigned int size,
813 					gfp_t priority)
814 {
815 	return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
816 }
817 
818 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
819 				     unsigned long data_len,
820 				     int max_page_order,
821 				     int *errcode,
822 				     gfp_t gfp_mask);
823 
824 /* Layout of fast clones : [skb1][skb2][fclone_ref] */
825 struct sk_buff_fclones {
826 	struct sk_buff	skb1;
827 
828 	struct sk_buff	skb2;
829 
830 	atomic_t	fclone_ref;
831 };
832 
833 /**
834  *	skb_fclone_busy - check if fclone is busy
835  *	@skb: buffer
836  *
837  * Returns true is skb is a fast clone, and its clone is not freed.
838  * Some drivers call skb_orphan() in their ndo_start_xmit(),
839  * so we also check that this didnt happen.
840  */
skb_fclone_busy(const struct sock * sk,const struct sk_buff * skb)841 static inline bool skb_fclone_busy(const struct sock *sk,
842 				   const struct sk_buff *skb)
843 {
844 	const struct sk_buff_fclones *fclones;
845 
846 	fclones = container_of(skb, struct sk_buff_fclones, skb1);
847 
848 	return skb->fclone == SKB_FCLONE_ORIG &&
849 	       atomic_read(&fclones->fclone_ref) > 1 &&
850 	       fclones->skb2.sk == sk;
851 }
852 
alloc_skb_fclone(unsigned int size,gfp_t priority)853 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
854 					       gfp_t priority)
855 {
856 	return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
857 }
858 
859 struct sk_buff *__alloc_skb_head(gfp_t priority, int node);
alloc_skb_head(gfp_t priority)860 static inline struct sk_buff *alloc_skb_head(gfp_t priority)
861 {
862 	return __alloc_skb_head(priority, -1);
863 }
864 
865 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
866 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
867 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
868 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
869 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
870 				   gfp_t gfp_mask, bool fclone);
__pskb_copy(struct sk_buff * skb,int headroom,gfp_t gfp_mask)871 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
872 					  gfp_t gfp_mask)
873 {
874 	return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
875 }
876 
877 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
878 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
879 				     unsigned int headroom);
880 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
881 				int newtailroom, gfp_t priority);
882 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
883 			int offset, int len);
884 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset,
885 		 int len);
886 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
887 int skb_pad(struct sk_buff *skb, int pad);
888 #define dev_kfree_skb(a)	consume_skb(a)
889 
890 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
891 			    int getfrag(void *from, char *to, int offset,
892 					int len, int odd, struct sk_buff *skb),
893 			    void *from, int length);
894 
895 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
896 			 int offset, size_t size);
897 
898 struct skb_seq_state {
899 	__u32		lower_offset;
900 	__u32		upper_offset;
901 	__u32		frag_idx;
902 	__u32		stepped_offset;
903 	struct sk_buff	*root_skb;
904 	struct sk_buff	*cur_skb;
905 	__u8		*frag_data;
906 };
907 
908 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
909 			  unsigned int to, struct skb_seq_state *st);
910 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
911 			  struct skb_seq_state *st);
912 void skb_abort_seq_read(struct skb_seq_state *st);
913 
914 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
915 			   unsigned int to, struct ts_config *config);
916 
917 /*
918  * Packet hash types specify the type of hash in skb_set_hash.
919  *
920  * Hash types refer to the protocol layer addresses which are used to
921  * construct a packet's hash. The hashes are used to differentiate or identify
922  * flows of the protocol layer for the hash type. Hash types are either
923  * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
924  *
925  * Properties of hashes:
926  *
927  * 1) Two packets in different flows have different hash values
928  * 2) Two packets in the same flow should have the same hash value
929  *
930  * A hash at a higher layer is considered to be more specific. A driver should
931  * set the most specific hash possible.
932  *
933  * A driver cannot indicate a more specific hash than the layer at which a hash
934  * was computed. For instance an L3 hash cannot be set as an L4 hash.
935  *
936  * A driver may indicate a hash level which is less specific than the
937  * actual layer the hash was computed on. For instance, a hash computed
938  * at L4 may be considered an L3 hash. This should only be done if the
939  * driver can't unambiguously determine that the HW computed the hash at
940  * the higher layer. Note that the "should" in the second property above
941  * permits this.
942  */
943 enum pkt_hash_types {
944 	PKT_HASH_TYPE_NONE,	/* Undefined type */
945 	PKT_HASH_TYPE_L2,	/* Input: src_MAC, dest_MAC */
946 	PKT_HASH_TYPE_L3,	/* Input: src_IP, dst_IP */
947 	PKT_HASH_TYPE_L4,	/* Input: src_IP, dst_IP, src_port, dst_port */
948 };
949 
skb_clear_hash(struct sk_buff * skb)950 static inline void skb_clear_hash(struct sk_buff *skb)
951 {
952 	skb->hash = 0;
953 	skb->sw_hash = 0;
954 	skb->l4_hash = 0;
955 }
956 
skb_clear_hash_if_not_l4(struct sk_buff * skb)957 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
958 {
959 	if (!skb->l4_hash)
960 		skb_clear_hash(skb);
961 }
962 
963 static inline void
__skb_set_hash(struct sk_buff * skb,__u32 hash,bool is_sw,bool is_l4)964 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
965 {
966 	skb->l4_hash = is_l4;
967 	skb->sw_hash = is_sw;
968 	skb->hash = hash;
969 }
970 
971 static inline void
skb_set_hash(struct sk_buff * skb,__u32 hash,enum pkt_hash_types type)972 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
973 {
974 	/* Used by drivers to set hash from HW */
975 	__skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
976 }
977 
978 static inline void
__skb_set_sw_hash(struct sk_buff * skb,__u32 hash,bool is_l4)979 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
980 {
981 	__skb_set_hash(skb, hash, true, is_l4);
982 }
983 
984 void __skb_get_hash(struct sk_buff *skb);
985 u32 skb_get_poff(const struct sk_buff *skb);
986 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
987 		   const struct flow_keys *keys, int hlen);
988 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
989 			    void *data, int hlen_proto);
990 
skb_flow_get_ports(const struct sk_buff * skb,int thoff,u8 ip_proto)991 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
992 					int thoff, u8 ip_proto)
993 {
994 	return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
995 }
996 
997 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
998 			     const struct flow_dissector_key *key,
999 			     unsigned int key_count);
1000 
1001 bool __skb_flow_dissect(const struct sk_buff *skb,
1002 			struct flow_dissector *flow_dissector,
1003 			void *target_container,
1004 			void *data, __be16 proto, int nhoff, int hlen,
1005 			unsigned int flags);
1006 
skb_flow_dissect(const struct sk_buff * skb,struct flow_dissector * flow_dissector,void * target_container,unsigned int flags)1007 static inline bool skb_flow_dissect(const struct sk_buff *skb,
1008 				    struct flow_dissector *flow_dissector,
1009 				    void *target_container, unsigned int flags)
1010 {
1011 	return __skb_flow_dissect(skb, flow_dissector, target_container,
1012 				  NULL, 0, 0, 0, flags);
1013 }
1014 
skb_flow_dissect_flow_keys(const struct sk_buff * skb,struct flow_keys * flow,unsigned int flags)1015 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
1016 					      struct flow_keys *flow,
1017 					      unsigned int flags)
1018 {
1019 	memset(flow, 0, sizeof(*flow));
1020 	return __skb_flow_dissect(skb, &flow_keys_dissector, flow,
1021 				  NULL, 0, 0, 0, flags);
1022 }
1023 
skb_flow_dissect_flow_keys_buf(struct flow_keys * flow,void * data,__be16 proto,int nhoff,int hlen,unsigned int flags)1024 static inline bool skb_flow_dissect_flow_keys_buf(struct flow_keys *flow,
1025 						  void *data, __be16 proto,
1026 						  int nhoff, int hlen,
1027 						  unsigned int flags)
1028 {
1029 	memset(flow, 0, sizeof(*flow));
1030 	return __skb_flow_dissect(NULL, &flow_keys_buf_dissector, flow,
1031 				  data, proto, nhoff, hlen, flags);
1032 }
1033 
skb_get_hash(struct sk_buff * skb)1034 static inline __u32 skb_get_hash(struct sk_buff *skb)
1035 {
1036 	if (!skb->l4_hash && !skb->sw_hash)
1037 		__skb_get_hash(skb);
1038 
1039 	return skb->hash;
1040 }
1041 
1042 __u32 __skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6);
1043 
skb_get_hash_flowi6(struct sk_buff * skb,const struct flowi6 * fl6)1044 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
1045 {
1046 	if (!skb->l4_hash && !skb->sw_hash) {
1047 		struct flow_keys keys;
1048 		__u32 hash = __get_hash_from_flowi6(fl6, &keys);
1049 
1050 		__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1051 	}
1052 
1053 	return skb->hash;
1054 }
1055 
1056 __u32 __skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl);
1057 
skb_get_hash_flowi4(struct sk_buff * skb,const struct flowi4 * fl4)1058 static inline __u32 skb_get_hash_flowi4(struct sk_buff *skb, const struct flowi4 *fl4)
1059 {
1060 	if (!skb->l4_hash && !skb->sw_hash) {
1061 		struct flow_keys keys;
1062 		__u32 hash = __get_hash_from_flowi4(fl4, &keys);
1063 
1064 		__skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1065 	}
1066 
1067 	return skb->hash;
1068 }
1069 
1070 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb);
1071 
skb_get_hash_raw(const struct sk_buff * skb)1072 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
1073 {
1074 	return skb->hash;
1075 }
1076 
skb_copy_hash(struct sk_buff * to,const struct sk_buff * from)1077 static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
1078 {
1079 	to->hash = from->hash;
1080 	to->sw_hash = from->sw_hash;
1081 	to->l4_hash = from->l4_hash;
1082 };
1083 
skb_sender_cpu_clear(struct sk_buff * skb)1084 static inline void skb_sender_cpu_clear(struct sk_buff *skb)
1085 {
1086 #ifdef CONFIG_XPS
1087 	skb->sender_cpu = 0;
1088 #endif
1089 }
1090 
1091 #ifdef NET_SKBUFF_DATA_USES_OFFSET
skb_end_pointer(const struct sk_buff * skb)1092 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1093 {
1094 	return skb->head + skb->end;
1095 }
1096 
skb_end_offset(const struct sk_buff * skb)1097 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1098 {
1099 	return skb->end;
1100 }
1101 #else
skb_end_pointer(const struct sk_buff * skb)1102 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1103 {
1104 	return skb->end;
1105 }
1106 
skb_end_offset(const struct sk_buff * skb)1107 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1108 {
1109 	return skb->end - skb->head;
1110 }
1111 #endif
1112 
1113 /* Internal */
1114 #define skb_shinfo(SKB)	((struct skb_shared_info *)(skb_end_pointer(SKB)))
1115 
skb_hwtstamps(struct sk_buff * skb)1116 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
1117 {
1118 	return &skb_shinfo(skb)->hwtstamps;
1119 }
1120 
1121 /**
1122  *	skb_queue_empty - check if a queue is empty
1123  *	@list: queue head
1124  *
1125  *	Returns true if the queue is empty, false otherwise.
1126  */
skb_queue_empty(const struct sk_buff_head * list)1127 static inline int skb_queue_empty(const struct sk_buff_head *list)
1128 {
1129 	return list->next == (const struct sk_buff *) list;
1130 }
1131 
1132 /**
1133  *	skb_queue_is_last - check if skb is the last entry in the queue
1134  *	@list: queue head
1135  *	@skb: buffer
1136  *
1137  *	Returns true if @skb is the last buffer on the list.
1138  */
skb_queue_is_last(const struct sk_buff_head * list,const struct sk_buff * skb)1139 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
1140 				     const struct sk_buff *skb)
1141 {
1142 	return skb->next == (const struct sk_buff *) list;
1143 }
1144 
1145 /**
1146  *	skb_queue_is_first - check if skb is the first entry in the queue
1147  *	@list: queue head
1148  *	@skb: buffer
1149  *
1150  *	Returns true if @skb is the first buffer on the list.
1151  */
skb_queue_is_first(const struct sk_buff_head * list,const struct sk_buff * skb)1152 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
1153 				      const struct sk_buff *skb)
1154 {
1155 	return skb->prev == (const struct sk_buff *) list;
1156 }
1157 
1158 /**
1159  *	skb_queue_next - return the next packet in the queue
1160  *	@list: queue head
1161  *	@skb: current buffer
1162  *
1163  *	Return the next packet in @list after @skb.  It is only valid to
1164  *	call this if skb_queue_is_last() evaluates to false.
1165  */
skb_queue_next(const struct sk_buff_head * list,const struct sk_buff * skb)1166 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
1167 					     const struct sk_buff *skb)
1168 {
1169 	/* This BUG_ON may seem severe, but if we just return then we
1170 	 * are going to dereference garbage.
1171 	 */
1172 	BUG_ON(skb_queue_is_last(list, skb));
1173 	return skb->next;
1174 }
1175 
1176 /**
1177  *	skb_queue_prev - return the prev packet in the queue
1178  *	@list: queue head
1179  *	@skb: current buffer
1180  *
1181  *	Return the prev packet in @list before @skb.  It is only valid to
1182  *	call this if skb_queue_is_first() evaluates to false.
1183  */
skb_queue_prev(const struct sk_buff_head * list,const struct sk_buff * skb)1184 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
1185 					     const struct sk_buff *skb)
1186 {
1187 	/* This BUG_ON may seem severe, but if we just return then we
1188 	 * are going to dereference garbage.
1189 	 */
1190 	BUG_ON(skb_queue_is_first(list, skb));
1191 	return skb->prev;
1192 }
1193 
1194 /**
1195  *	skb_get - reference buffer
1196  *	@skb: buffer to reference
1197  *
1198  *	Makes another reference to a socket buffer and returns a pointer
1199  *	to the buffer.
1200  */
skb_get(struct sk_buff * skb)1201 static inline struct sk_buff *skb_get(struct sk_buff *skb)
1202 {
1203 	atomic_inc(&skb->users);
1204 	return skb;
1205 }
1206 
1207 /*
1208  * If users == 1, we are the only owner and are can avoid redundant
1209  * atomic change.
1210  */
1211 
1212 /**
1213  *	skb_cloned - is the buffer a clone
1214  *	@skb: buffer to check
1215  *
1216  *	Returns true if the buffer was generated with skb_clone() and is
1217  *	one of multiple shared copies of the buffer. Cloned buffers are
1218  *	shared data so must not be written to under normal circumstances.
1219  */
skb_cloned(const struct sk_buff * skb)1220 static inline int skb_cloned(const struct sk_buff *skb)
1221 {
1222 	return skb->cloned &&
1223 	       (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
1224 }
1225 
skb_unclone(struct sk_buff * skb,gfp_t pri)1226 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1227 {
1228 	might_sleep_if(gfpflags_allow_blocking(pri));
1229 
1230 	if (skb_cloned(skb))
1231 		return pskb_expand_head(skb, 0, 0, pri);
1232 
1233 	return 0;
1234 }
1235 
1236 /**
1237  *	skb_header_cloned - is the header a clone
1238  *	@skb: buffer to check
1239  *
1240  *	Returns true if modifying the header part of the buffer requires
1241  *	the data to be copied.
1242  */
skb_header_cloned(const struct sk_buff * skb)1243 static inline int skb_header_cloned(const struct sk_buff *skb)
1244 {
1245 	int dataref;
1246 
1247 	if (!skb->cloned)
1248 		return 0;
1249 
1250 	dataref = atomic_read(&skb_shinfo(skb)->dataref);
1251 	dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
1252 	return dataref != 1;
1253 }
1254 
1255 /**
1256  *	skb_header_release - release reference to header
1257  *	@skb: buffer to operate on
1258  *
1259  *	Drop a reference to the header part of the buffer.  This is done
1260  *	by acquiring a payload reference.  You must not read from the header
1261  *	part of skb->data after this.
1262  *	Note : Check if you can use __skb_header_release() instead.
1263  */
skb_header_release(struct sk_buff * skb)1264 static inline void skb_header_release(struct sk_buff *skb)
1265 {
1266 	BUG_ON(skb->nohdr);
1267 	skb->nohdr = 1;
1268 	atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
1269 }
1270 
1271 /**
1272  *	__skb_header_release - release reference to header
1273  *	@skb: buffer to operate on
1274  *
1275  *	Variant of skb_header_release() assuming skb is private to caller.
1276  *	We can avoid one atomic operation.
1277  */
__skb_header_release(struct sk_buff * skb)1278 static inline void __skb_header_release(struct sk_buff *skb)
1279 {
1280 	skb->nohdr = 1;
1281 	atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
1282 }
1283 
1284 
1285 /**
1286  *	skb_shared - is the buffer shared
1287  *	@skb: buffer to check
1288  *
1289  *	Returns true if more than one person has a reference to this
1290  *	buffer.
1291  */
skb_shared(const struct sk_buff * skb)1292 static inline int skb_shared(const struct sk_buff *skb)
1293 {
1294 	return atomic_read(&skb->users) != 1;
1295 }
1296 
1297 /**
1298  *	skb_share_check - check if buffer is shared and if so clone it
1299  *	@skb: buffer to check
1300  *	@pri: priority for memory allocation
1301  *
1302  *	If the buffer is shared the buffer is cloned and the old copy
1303  *	drops a reference. A new clone with a single reference is returned.
1304  *	If the buffer is not shared the original buffer is returned. When
1305  *	being called from interrupt status or with spinlocks held pri must
1306  *	be GFP_ATOMIC.
1307  *
1308  *	NULL is returned on a memory allocation failure.
1309  */
skb_share_check(struct sk_buff * skb,gfp_t pri)1310 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
1311 {
1312 	might_sleep_if(gfpflags_allow_blocking(pri));
1313 	if (skb_shared(skb)) {
1314 		struct sk_buff *nskb = skb_clone(skb, pri);
1315 
1316 		if (likely(nskb))
1317 			consume_skb(skb);
1318 		else
1319 			kfree_skb(skb);
1320 		skb = nskb;
1321 	}
1322 	return skb;
1323 }
1324 
1325 /*
1326  *	Copy shared buffers into a new sk_buff. We effectively do COW on
1327  *	packets to handle cases where we have a local reader and forward
1328  *	and a couple of other messy ones. The normal one is tcpdumping
1329  *	a packet thats being forwarded.
1330  */
1331 
1332 /**
1333  *	skb_unshare - make a copy of a shared buffer
1334  *	@skb: buffer to check
1335  *	@pri: priority for memory allocation
1336  *
1337  *	If the socket buffer is a clone then this function creates a new
1338  *	copy of the data, drops a reference count on the old copy and returns
1339  *	the new copy with the reference count at 1. If the buffer is not a clone
1340  *	the original buffer is returned. When called with a spinlock held or
1341  *	from interrupt state @pri must be %GFP_ATOMIC
1342  *
1343  *	%NULL is returned on a memory allocation failure.
1344  */
skb_unshare(struct sk_buff * skb,gfp_t pri)1345 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
1346 					  gfp_t pri)
1347 {
1348 	might_sleep_if(gfpflags_allow_blocking(pri));
1349 	if (skb_cloned(skb)) {
1350 		struct sk_buff *nskb = skb_copy(skb, pri);
1351 
1352 		/* Free our shared copy */
1353 		if (likely(nskb))
1354 			consume_skb(skb);
1355 		else
1356 			kfree_skb(skb);
1357 		skb = nskb;
1358 	}
1359 	return skb;
1360 }
1361 
1362 /**
1363  *	skb_peek - peek at the head of an &sk_buff_head
1364  *	@list_: list to peek at
1365  *
1366  *	Peek an &sk_buff. Unlike most other operations you _MUST_
1367  *	be careful with this one. A peek leaves the buffer on the
1368  *	list and someone else may run off with it. You must hold
1369  *	the appropriate locks or have a private queue to do this.
1370  *
1371  *	Returns %NULL for an empty list or a pointer to the head element.
1372  *	The reference count is not incremented and the reference is therefore
1373  *	volatile. Use with caution.
1374  */
skb_peek(const struct sk_buff_head * list_)1375 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
1376 {
1377 	struct sk_buff *skb = list_->next;
1378 
1379 	if (skb == (struct sk_buff *)list_)
1380 		skb = NULL;
1381 	return skb;
1382 }
1383 
1384 /**
1385  *	skb_peek_next - peek skb following the given one from a queue
1386  *	@skb: skb to start from
1387  *	@list_: list to peek at
1388  *
1389  *	Returns %NULL when the end of the list is met or a pointer to the
1390  *	next element. The reference count is not incremented and the
1391  *	reference is therefore volatile. Use with caution.
1392  */
skb_peek_next(struct sk_buff * skb,const struct sk_buff_head * list_)1393 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
1394 		const struct sk_buff_head *list_)
1395 {
1396 	struct sk_buff *next = skb->next;
1397 
1398 	if (next == (struct sk_buff *)list_)
1399 		next = NULL;
1400 	return next;
1401 }
1402 
1403 /**
1404  *	skb_peek_tail - peek at the tail of an &sk_buff_head
1405  *	@list_: list to peek at
1406  *
1407  *	Peek an &sk_buff. Unlike most other operations you _MUST_
1408  *	be careful with this one. A peek leaves the buffer on the
1409  *	list and someone else may run off with it. You must hold
1410  *	the appropriate locks or have a private queue to do this.
1411  *
1412  *	Returns %NULL for an empty list or a pointer to the tail element.
1413  *	The reference count is not incremented and the reference is therefore
1414  *	volatile. Use with caution.
1415  */
skb_peek_tail(const struct sk_buff_head * list_)1416 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
1417 {
1418 	struct sk_buff *skb = list_->prev;
1419 
1420 	if (skb == (struct sk_buff *)list_)
1421 		skb = NULL;
1422 	return skb;
1423 
1424 }
1425 
1426 /**
1427  *	skb_queue_len	- get queue length
1428  *	@list_: list to measure
1429  *
1430  *	Return the length of an &sk_buff queue.
1431  */
skb_queue_len(const struct sk_buff_head * list_)1432 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
1433 {
1434 	return list_->qlen;
1435 }
1436 
1437 /**
1438  *	__skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
1439  *	@list: queue to initialize
1440  *
1441  *	This initializes only the list and queue length aspects of
1442  *	an sk_buff_head object.  This allows to initialize the list
1443  *	aspects of an sk_buff_head without reinitializing things like
1444  *	the spinlock.  It can also be used for on-stack sk_buff_head
1445  *	objects where the spinlock is known to not be used.
1446  */
__skb_queue_head_init(struct sk_buff_head * list)1447 static inline void __skb_queue_head_init(struct sk_buff_head *list)
1448 {
1449 	list->prev = list->next = (struct sk_buff *)list;
1450 	list->qlen = 0;
1451 }
1452 
1453 /*
1454  * This function creates a split out lock class for each invocation;
1455  * this is needed for now since a whole lot of users of the skb-queue
1456  * infrastructure in drivers have different locking usage (in hardirq)
1457  * than the networking core (in softirq only). In the long run either the
1458  * network layer or drivers should need annotation to consolidate the
1459  * main types of usage into 3 classes.
1460  */
skb_queue_head_init(struct sk_buff_head * list)1461 static inline void skb_queue_head_init(struct sk_buff_head *list)
1462 {
1463 	spin_lock_init(&list->lock);
1464 	__skb_queue_head_init(list);
1465 }
1466 
skb_queue_head_init_class(struct sk_buff_head * list,struct lock_class_key * class)1467 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
1468 		struct lock_class_key *class)
1469 {
1470 	skb_queue_head_init(list);
1471 	lockdep_set_class(&list->lock, class);
1472 }
1473 
1474 /*
1475  *	Insert an sk_buff on a list.
1476  *
1477  *	The "__skb_xxxx()" functions are the non-atomic ones that
1478  *	can only be called with interrupts disabled.
1479  */
1480 void skb_insert(struct sk_buff *old, struct sk_buff *newsk,
1481 		struct sk_buff_head *list);
__skb_insert(struct sk_buff * newsk,struct sk_buff * prev,struct sk_buff * next,struct sk_buff_head * list)1482 static inline void __skb_insert(struct sk_buff *newsk,
1483 				struct sk_buff *prev, struct sk_buff *next,
1484 				struct sk_buff_head *list)
1485 {
1486 	newsk->next = next;
1487 	newsk->prev = prev;
1488 	next->prev  = prev->next = newsk;
1489 	list->qlen++;
1490 }
1491 
__skb_queue_splice(const struct sk_buff_head * list,struct sk_buff * prev,struct sk_buff * next)1492 static inline void __skb_queue_splice(const struct sk_buff_head *list,
1493 				      struct sk_buff *prev,
1494 				      struct sk_buff *next)
1495 {
1496 	struct sk_buff *first = list->next;
1497 	struct sk_buff *last = list->prev;
1498 
1499 	first->prev = prev;
1500 	prev->next = first;
1501 
1502 	last->next = next;
1503 	next->prev = last;
1504 }
1505 
1506 /**
1507  *	skb_queue_splice - join two skb lists, this is designed for stacks
1508  *	@list: the new list to add
1509  *	@head: the place to add it in the first list
1510  */
skb_queue_splice(const struct sk_buff_head * list,struct sk_buff_head * head)1511 static inline void skb_queue_splice(const struct sk_buff_head *list,
1512 				    struct sk_buff_head *head)
1513 {
1514 	if (!skb_queue_empty(list)) {
1515 		__skb_queue_splice(list, (struct sk_buff *) head, head->next);
1516 		head->qlen += list->qlen;
1517 	}
1518 }
1519 
1520 /**
1521  *	skb_queue_splice_init - join two skb lists and reinitialise the emptied list
1522  *	@list: the new list to add
1523  *	@head: the place to add it in the first list
1524  *
1525  *	The list at @list is reinitialised
1526  */
skb_queue_splice_init(struct sk_buff_head * list,struct sk_buff_head * head)1527 static inline void skb_queue_splice_init(struct sk_buff_head *list,
1528 					 struct sk_buff_head *head)
1529 {
1530 	if (!skb_queue_empty(list)) {
1531 		__skb_queue_splice(list, (struct sk_buff *) head, head->next);
1532 		head->qlen += list->qlen;
1533 		__skb_queue_head_init(list);
1534 	}
1535 }
1536 
1537 /**
1538  *	skb_queue_splice_tail - join two skb lists, each list being a queue
1539  *	@list: the new list to add
1540  *	@head: the place to add it in the first list
1541  */
skb_queue_splice_tail(const struct sk_buff_head * list,struct sk_buff_head * head)1542 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1543 					 struct sk_buff_head *head)
1544 {
1545 	if (!skb_queue_empty(list)) {
1546 		__skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1547 		head->qlen += list->qlen;
1548 	}
1549 }
1550 
1551 /**
1552  *	skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
1553  *	@list: the new list to add
1554  *	@head: the place to add it in the first list
1555  *
1556  *	Each of the lists is a queue.
1557  *	The list at @list is reinitialised
1558  */
skb_queue_splice_tail_init(struct sk_buff_head * list,struct sk_buff_head * head)1559 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1560 					      struct sk_buff_head *head)
1561 {
1562 	if (!skb_queue_empty(list)) {
1563 		__skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1564 		head->qlen += list->qlen;
1565 		__skb_queue_head_init(list);
1566 	}
1567 }
1568 
1569 /**
1570  *	__skb_queue_after - queue a buffer at the list head
1571  *	@list: list to use
1572  *	@prev: place after this buffer
1573  *	@newsk: buffer to queue
1574  *
1575  *	Queue a buffer int the middle of a list. This function takes no locks
1576  *	and you must therefore hold required locks before calling it.
1577  *
1578  *	A buffer cannot be placed on two lists at the same time.
1579  */
__skb_queue_after(struct sk_buff_head * list,struct sk_buff * prev,struct sk_buff * newsk)1580 static inline void __skb_queue_after(struct sk_buff_head *list,
1581 				     struct sk_buff *prev,
1582 				     struct sk_buff *newsk)
1583 {
1584 	__skb_insert(newsk, prev, prev->next, list);
1585 }
1586 
1587 void skb_append(struct sk_buff *old, struct sk_buff *newsk,
1588 		struct sk_buff_head *list);
1589 
__skb_queue_before(struct sk_buff_head * list,struct sk_buff * next,struct sk_buff * newsk)1590 static inline void __skb_queue_before(struct sk_buff_head *list,
1591 				      struct sk_buff *next,
1592 				      struct sk_buff *newsk)
1593 {
1594 	__skb_insert(newsk, next->prev, next, list);
1595 }
1596 
1597 /**
1598  *	__skb_queue_head - queue a buffer at the list head
1599  *	@list: list to use
1600  *	@newsk: buffer to queue
1601  *
1602  *	Queue a buffer at the start of a list. This function takes no locks
1603  *	and you must therefore hold required locks before calling it.
1604  *
1605  *	A buffer cannot be placed on two lists at the same time.
1606  */
1607 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
__skb_queue_head(struct sk_buff_head * list,struct sk_buff * newsk)1608 static inline void __skb_queue_head(struct sk_buff_head *list,
1609 				    struct sk_buff *newsk)
1610 {
1611 	__skb_queue_after(list, (struct sk_buff *)list, newsk);
1612 }
1613 
1614 /**
1615  *	__skb_queue_tail - queue a buffer at the list tail
1616  *	@list: list to use
1617  *	@newsk: buffer to queue
1618  *
1619  *	Queue a buffer at the end of a list. This function takes no locks
1620  *	and you must therefore hold required locks before calling it.
1621  *
1622  *	A buffer cannot be placed on two lists at the same time.
1623  */
1624 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
__skb_queue_tail(struct sk_buff_head * list,struct sk_buff * newsk)1625 static inline void __skb_queue_tail(struct sk_buff_head *list,
1626 				   struct sk_buff *newsk)
1627 {
1628 	__skb_queue_before(list, (struct sk_buff *)list, newsk);
1629 }
1630 
1631 /*
1632  * remove sk_buff from list. _Must_ be called atomically, and with
1633  * the list known..
1634  */
1635 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
__skb_unlink(struct sk_buff * skb,struct sk_buff_head * list)1636 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1637 {
1638 	struct sk_buff *next, *prev;
1639 
1640 	list->qlen--;
1641 	next	   = skb->next;
1642 	prev	   = skb->prev;
1643 	skb->next  = skb->prev = NULL;
1644 	next->prev = prev;
1645 	prev->next = next;
1646 }
1647 
1648 /**
1649  *	__skb_dequeue - remove from the head of the queue
1650  *	@list: list to dequeue from
1651  *
1652  *	Remove the head of the list. This function does not take any locks
1653  *	so must be used with appropriate locks held only. The head item is
1654  *	returned or %NULL if the list is empty.
1655  */
1656 struct sk_buff *skb_dequeue(struct sk_buff_head *list);
__skb_dequeue(struct sk_buff_head * list)1657 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
1658 {
1659 	struct sk_buff *skb = skb_peek(list);
1660 	if (skb)
1661 		__skb_unlink(skb, list);
1662 	return skb;
1663 }
1664 
1665 /**
1666  *	__skb_dequeue_tail - remove from the tail of the queue
1667  *	@list: list to dequeue from
1668  *
1669  *	Remove the tail of the list. This function does not take any locks
1670  *	so must be used with appropriate locks held only. The tail item is
1671  *	returned or %NULL if the list is empty.
1672  */
1673 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
__skb_dequeue_tail(struct sk_buff_head * list)1674 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
1675 {
1676 	struct sk_buff *skb = skb_peek_tail(list);
1677 	if (skb)
1678 		__skb_unlink(skb, list);
1679 	return skb;
1680 }
1681 
1682 
skb_is_nonlinear(const struct sk_buff * skb)1683 static inline bool skb_is_nonlinear(const struct sk_buff *skb)
1684 {
1685 	return skb->data_len;
1686 }
1687 
skb_headlen(const struct sk_buff * skb)1688 static inline unsigned int skb_headlen(const struct sk_buff *skb)
1689 {
1690 	return skb->len - skb->data_len;
1691 }
1692 
skb_pagelen(const struct sk_buff * skb)1693 static inline int skb_pagelen(const struct sk_buff *skb)
1694 {
1695 	int i, len = 0;
1696 
1697 	for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--)
1698 		len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
1699 	return len + skb_headlen(skb);
1700 }
1701 
1702 /**
1703  * __skb_fill_page_desc - initialise a paged fragment in an skb
1704  * @skb: buffer containing fragment to be initialised
1705  * @i: paged fragment index to initialise
1706  * @page: the page to use for this fragment
1707  * @off: the offset to the data with @page
1708  * @size: the length of the data
1709  *
1710  * Initialises the @i'th fragment of @skb to point to &size bytes at
1711  * offset @off within @page.
1712  *
1713  * Does not take any additional reference on the fragment.
1714  */
__skb_fill_page_desc(struct sk_buff * skb,int i,struct page * page,int off,int size)1715 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
1716 					struct page *page, int off, int size)
1717 {
1718 	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1719 
1720 	/*
1721 	 * Propagate page pfmemalloc to the skb if we can. The problem is
1722 	 * that not all callers have unique ownership of the page but rely
1723 	 * on page_is_pfmemalloc doing the right thing(tm).
1724 	 */
1725 	frag->page.p		  = page;
1726 	frag->page_offset	  = off;
1727 	skb_frag_size_set(frag, size);
1728 
1729 	page = compound_head(page);
1730 	if (page_is_pfmemalloc(page))
1731 		skb->pfmemalloc	= true;
1732 }
1733 
1734 /**
1735  * skb_fill_page_desc - initialise a paged fragment in an skb
1736  * @skb: buffer containing fragment to be initialised
1737  * @i: paged fragment index to initialise
1738  * @page: the page to use for this fragment
1739  * @off: the offset to the data with @page
1740  * @size: the length of the data
1741  *
1742  * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
1743  * @skb to point to @size bytes at offset @off within @page. In
1744  * addition updates @skb such that @i is the last fragment.
1745  *
1746  * Does not take any additional reference on the fragment.
1747  */
skb_fill_page_desc(struct sk_buff * skb,int i,struct page * page,int off,int size)1748 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
1749 				      struct page *page, int off, int size)
1750 {
1751 	__skb_fill_page_desc(skb, i, page, off, size);
1752 	skb_shinfo(skb)->nr_frags = i + 1;
1753 }
1754 
1755 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
1756 		     int size, unsigned int truesize);
1757 
1758 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
1759 			  unsigned int truesize);
1760 
1761 #define SKB_PAGE_ASSERT(skb) 	BUG_ON(skb_shinfo(skb)->nr_frags)
1762 #define SKB_FRAG_ASSERT(skb) 	BUG_ON(skb_has_frag_list(skb))
1763 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
1764 
1765 #ifdef NET_SKBUFF_DATA_USES_OFFSET
skb_tail_pointer(const struct sk_buff * skb)1766 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1767 {
1768 	return skb->head + skb->tail;
1769 }
1770 
skb_reset_tail_pointer(struct sk_buff * skb)1771 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1772 {
1773 	skb->tail = skb->data - skb->head;
1774 }
1775 
skb_set_tail_pointer(struct sk_buff * skb,const int offset)1776 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1777 {
1778 	skb_reset_tail_pointer(skb);
1779 	skb->tail += offset;
1780 }
1781 
1782 #else /* NET_SKBUFF_DATA_USES_OFFSET */
skb_tail_pointer(const struct sk_buff * skb)1783 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1784 {
1785 	return skb->tail;
1786 }
1787 
skb_reset_tail_pointer(struct sk_buff * skb)1788 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1789 {
1790 	skb->tail = skb->data;
1791 }
1792 
skb_set_tail_pointer(struct sk_buff * skb,const int offset)1793 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1794 {
1795 	skb->tail = skb->data + offset;
1796 }
1797 
1798 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
1799 
1800 /*
1801  *	Add data to an sk_buff
1802  */
1803 unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
1804 unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
__skb_put(struct sk_buff * skb,unsigned int len)1805 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
1806 {
1807 	unsigned char *tmp = skb_tail_pointer(skb);
1808 	SKB_LINEAR_ASSERT(skb);
1809 	skb->tail += len;
1810 	skb->len  += len;
1811 	return tmp;
1812 }
1813 
1814 unsigned char *skb_push(struct sk_buff *skb, unsigned int len);
__skb_push(struct sk_buff * skb,unsigned int len)1815 static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
1816 {
1817 	skb->data -= len;
1818 	skb->len  += len;
1819 	return skb->data;
1820 }
1821 
1822 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
__skb_pull(struct sk_buff * skb,unsigned int len)1823 static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
1824 {
1825 	skb->len -= len;
1826 	BUG_ON(skb->len < skb->data_len);
1827 	return skb->data += len;
1828 }
1829 
skb_pull_inline(struct sk_buff * skb,unsigned int len)1830 static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
1831 {
1832 	return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1833 }
1834 
1835 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
1836 
__pskb_pull(struct sk_buff * skb,unsigned int len)1837 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
1838 {
1839 	if (len > skb_headlen(skb) &&
1840 	    !__pskb_pull_tail(skb, len - skb_headlen(skb)))
1841 		return NULL;
1842 	skb->len -= len;
1843 	return skb->data += len;
1844 }
1845 
pskb_pull(struct sk_buff * skb,unsigned int len)1846 static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len)
1847 {
1848 	return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
1849 }
1850 
pskb_may_pull(struct sk_buff * skb,unsigned int len)1851 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
1852 {
1853 	if (likely(len <= skb_headlen(skb)))
1854 		return 1;
1855 	if (unlikely(len > skb->len))
1856 		return 0;
1857 	return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
1858 }
1859 
1860 /**
1861  *	skb_headroom - bytes at buffer head
1862  *	@skb: buffer to check
1863  *
1864  *	Return the number of bytes of free space at the head of an &sk_buff.
1865  */
skb_headroom(const struct sk_buff * skb)1866 static inline unsigned int skb_headroom(const struct sk_buff *skb)
1867 {
1868 	return skb->data - skb->head;
1869 }
1870 
1871 /**
1872  *	skb_tailroom - bytes at buffer end
1873  *	@skb: buffer to check
1874  *
1875  *	Return the number of bytes of free space at the tail of an sk_buff
1876  */
skb_tailroom(const struct sk_buff * skb)1877 static inline int skb_tailroom(const struct sk_buff *skb)
1878 {
1879 	return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
1880 }
1881 
1882 /**
1883  *	skb_availroom - bytes at buffer end
1884  *	@skb: buffer to check
1885  *
1886  *	Return the number of bytes of free space at the tail of an sk_buff
1887  *	allocated by sk_stream_alloc()
1888  */
skb_availroom(const struct sk_buff * skb)1889 static inline int skb_availroom(const struct sk_buff *skb)
1890 {
1891 	if (skb_is_nonlinear(skb))
1892 		return 0;
1893 
1894 	return skb->end - skb->tail - skb->reserved_tailroom;
1895 }
1896 
1897 /**
1898  *	skb_reserve - adjust headroom
1899  *	@skb: buffer to alter
1900  *	@len: bytes to move
1901  *
1902  *	Increase the headroom of an empty &sk_buff by reducing the tail
1903  *	room. This is only allowed for an empty buffer.
1904  */
skb_reserve(struct sk_buff * skb,int len)1905 static inline void skb_reserve(struct sk_buff *skb, int len)
1906 {
1907 	skb->data += len;
1908 	skb->tail += len;
1909 }
1910 
1911 /**
1912  *	skb_tailroom_reserve - adjust reserved_tailroom
1913  *	@skb: buffer to alter
1914  *	@mtu: maximum amount of headlen permitted
1915  *	@needed_tailroom: minimum amount of reserved_tailroom
1916  *
1917  *	Set reserved_tailroom so that headlen can be as large as possible but
1918  *	not larger than mtu and tailroom cannot be smaller than
1919  *	needed_tailroom.
1920  *	The required headroom should already have been reserved before using
1921  *	this function.
1922  */
skb_tailroom_reserve(struct sk_buff * skb,unsigned int mtu,unsigned int needed_tailroom)1923 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
1924 					unsigned int needed_tailroom)
1925 {
1926 	SKB_LINEAR_ASSERT(skb);
1927 	if (mtu < skb_tailroom(skb) - needed_tailroom)
1928 		/* use at most mtu */
1929 		skb->reserved_tailroom = skb_tailroom(skb) - mtu;
1930 	else
1931 		/* use up to all available space */
1932 		skb->reserved_tailroom = needed_tailroom;
1933 }
1934 
1935 #define ENCAP_TYPE_ETHER	0
1936 #define ENCAP_TYPE_IPPROTO	1
1937 
skb_set_inner_protocol(struct sk_buff * skb,__be16 protocol)1938 static inline void skb_set_inner_protocol(struct sk_buff *skb,
1939 					  __be16 protocol)
1940 {
1941 	skb->inner_protocol = protocol;
1942 	skb->inner_protocol_type = ENCAP_TYPE_ETHER;
1943 }
1944 
skb_set_inner_ipproto(struct sk_buff * skb,__u8 ipproto)1945 static inline void skb_set_inner_ipproto(struct sk_buff *skb,
1946 					 __u8 ipproto)
1947 {
1948 	skb->inner_ipproto = ipproto;
1949 	skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
1950 }
1951 
skb_reset_inner_headers(struct sk_buff * skb)1952 static inline void skb_reset_inner_headers(struct sk_buff *skb)
1953 {
1954 	skb->inner_mac_header = skb->mac_header;
1955 	skb->inner_network_header = skb->network_header;
1956 	skb->inner_transport_header = skb->transport_header;
1957 }
1958 
skb_reset_mac_len(struct sk_buff * skb)1959 static inline void skb_reset_mac_len(struct sk_buff *skb)
1960 {
1961 	skb->mac_len = skb->network_header - skb->mac_header;
1962 }
1963 
skb_inner_transport_header(const struct sk_buff * skb)1964 static inline unsigned char *skb_inner_transport_header(const struct sk_buff
1965 							*skb)
1966 {
1967 	return skb->head + skb->inner_transport_header;
1968 }
1969 
skb_reset_inner_transport_header(struct sk_buff * skb)1970 static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
1971 {
1972 	skb->inner_transport_header = skb->data - skb->head;
1973 }
1974 
skb_set_inner_transport_header(struct sk_buff * skb,const int offset)1975 static inline void skb_set_inner_transport_header(struct sk_buff *skb,
1976 						   const int offset)
1977 {
1978 	skb_reset_inner_transport_header(skb);
1979 	skb->inner_transport_header += offset;
1980 }
1981 
skb_inner_network_header(const struct sk_buff * skb)1982 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
1983 {
1984 	return skb->head + skb->inner_network_header;
1985 }
1986 
skb_reset_inner_network_header(struct sk_buff * skb)1987 static inline void skb_reset_inner_network_header(struct sk_buff *skb)
1988 {
1989 	skb->inner_network_header = skb->data - skb->head;
1990 }
1991 
skb_set_inner_network_header(struct sk_buff * skb,const int offset)1992 static inline void skb_set_inner_network_header(struct sk_buff *skb,
1993 						const int offset)
1994 {
1995 	skb_reset_inner_network_header(skb);
1996 	skb->inner_network_header += offset;
1997 }
1998 
skb_inner_mac_header(const struct sk_buff * skb)1999 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
2000 {
2001 	return skb->head + skb->inner_mac_header;
2002 }
2003 
skb_reset_inner_mac_header(struct sk_buff * skb)2004 static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
2005 {
2006 	skb->inner_mac_header = skb->data - skb->head;
2007 }
2008 
skb_set_inner_mac_header(struct sk_buff * skb,const int offset)2009 static inline void skb_set_inner_mac_header(struct sk_buff *skb,
2010 					    const int offset)
2011 {
2012 	skb_reset_inner_mac_header(skb);
2013 	skb->inner_mac_header += offset;
2014 }
skb_transport_header_was_set(const struct sk_buff * skb)2015 static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
2016 {
2017 	return skb->transport_header != (typeof(skb->transport_header))~0U;
2018 }
2019 
skb_transport_header(const struct sk_buff * skb)2020 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
2021 {
2022 	return skb->head + skb->transport_header;
2023 }
2024 
skb_reset_transport_header(struct sk_buff * skb)2025 static inline void skb_reset_transport_header(struct sk_buff *skb)
2026 {
2027 	skb->transport_header = skb->data - skb->head;
2028 }
2029 
skb_set_transport_header(struct sk_buff * skb,const int offset)2030 static inline void skb_set_transport_header(struct sk_buff *skb,
2031 					    const int offset)
2032 {
2033 	skb_reset_transport_header(skb);
2034 	skb->transport_header += offset;
2035 }
2036 
skb_network_header(const struct sk_buff * skb)2037 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
2038 {
2039 	return skb->head + skb->network_header;
2040 }
2041 
skb_reset_network_header(struct sk_buff * skb)2042 static inline void skb_reset_network_header(struct sk_buff *skb)
2043 {
2044 	skb->network_header = skb->data - skb->head;
2045 }
2046 
skb_set_network_header(struct sk_buff * skb,const int offset)2047 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
2048 {
2049 	skb_reset_network_header(skb);
2050 	skb->network_header += offset;
2051 }
2052 
skb_mac_header(const struct sk_buff * skb)2053 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
2054 {
2055 	return skb->head + skb->mac_header;
2056 }
2057 
skb_mac_header_was_set(const struct sk_buff * skb)2058 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
2059 {
2060 	return skb->mac_header != (typeof(skb->mac_header))~0U;
2061 }
2062 
skb_reset_mac_header(struct sk_buff * skb)2063 static inline void skb_reset_mac_header(struct sk_buff *skb)
2064 {
2065 	skb->mac_header = skb->data - skb->head;
2066 }
2067 
skb_set_mac_header(struct sk_buff * skb,const int offset)2068 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
2069 {
2070 	skb_reset_mac_header(skb);
2071 	skb->mac_header += offset;
2072 }
2073 
skb_pop_mac_header(struct sk_buff * skb)2074 static inline void skb_pop_mac_header(struct sk_buff *skb)
2075 {
2076 	skb->mac_header = skb->network_header;
2077 }
2078 
skb_probe_transport_header(struct sk_buff * skb,const int offset_hint)2079 static inline void skb_probe_transport_header(struct sk_buff *skb,
2080 					      const int offset_hint)
2081 {
2082 	struct flow_keys keys;
2083 
2084 	if (skb_transport_header_was_set(skb))
2085 		return;
2086 	else if (skb_flow_dissect_flow_keys(skb, &keys, 0))
2087 		skb_set_transport_header(skb, keys.control.thoff);
2088 	else
2089 		skb_set_transport_header(skb, offset_hint);
2090 }
2091 
skb_mac_header_rebuild(struct sk_buff * skb)2092 static inline void skb_mac_header_rebuild(struct sk_buff *skb)
2093 {
2094 	if (skb_mac_header_was_set(skb)) {
2095 		const unsigned char *old_mac = skb_mac_header(skb);
2096 
2097 		skb_set_mac_header(skb, -skb->mac_len);
2098 		memmove(skb_mac_header(skb), old_mac, skb->mac_len);
2099 	}
2100 }
2101 
skb_checksum_start_offset(const struct sk_buff * skb)2102 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
2103 {
2104 	return skb->csum_start - skb_headroom(skb);
2105 }
2106 
skb_transport_offset(const struct sk_buff * skb)2107 static inline int skb_transport_offset(const struct sk_buff *skb)
2108 {
2109 	return skb_transport_header(skb) - skb->data;
2110 }
2111 
skb_network_header_len(const struct sk_buff * skb)2112 static inline u32 skb_network_header_len(const struct sk_buff *skb)
2113 {
2114 	return skb->transport_header - skb->network_header;
2115 }
2116 
skb_inner_network_header_len(const struct sk_buff * skb)2117 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
2118 {
2119 	return skb->inner_transport_header - skb->inner_network_header;
2120 }
2121 
skb_network_offset(const struct sk_buff * skb)2122 static inline int skb_network_offset(const struct sk_buff *skb)
2123 {
2124 	return skb_network_header(skb) - skb->data;
2125 }
2126 
skb_inner_network_offset(const struct sk_buff * skb)2127 static inline int skb_inner_network_offset(const struct sk_buff *skb)
2128 {
2129 	return skb_inner_network_header(skb) - skb->data;
2130 }
2131 
pskb_network_may_pull(struct sk_buff * skb,unsigned int len)2132 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
2133 {
2134 	return pskb_may_pull(skb, skb_network_offset(skb) + len);
2135 }
2136 
2137 /*
2138  * CPUs often take a performance hit when accessing unaligned memory
2139  * locations. The actual performance hit varies, it can be small if the
2140  * hardware handles it or large if we have to take an exception and fix it
2141  * in software.
2142  *
2143  * Since an ethernet header is 14 bytes network drivers often end up with
2144  * the IP header at an unaligned offset. The IP header can be aligned by
2145  * shifting the start of the packet by 2 bytes. Drivers should do this
2146  * with:
2147  *
2148  * skb_reserve(skb, NET_IP_ALIGN);
2149  *
2150  * The downside to this alignment of the IP header is that the DMA is now
2151  * unaligned. On some architectures the cost of an unaligned DMA is high
2152  * and this cost outweighs the gains made by aligning the IP header.
2153  *
2154  * Since this trade off varies between architectures, we allow NET_IP_ALIGN
2155  * to be overridden.
2156  */
2157 #ifndef NET_IP_ALIGN
2158 #define NET_IP_ALIGN	2
2159 #endif
2160 
2161 /*
2162  * The networking layer reserves some headroom in skb data (via
2163  * dev_alloc_skb). This is used to avoid having to reallocate skb data when
2164  * the header has to grow. In the default case, if the header has to grow
2165  * 32 bytes or less we avoid the reallocation.
2166  *
2167  * Unfortunately this headroom changes the DMA alignment of the resulting
2168  * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
2169  * on some architectures. An architecture can override this value,
2170  * perhaps setting it to a cacheline in size (since that will maintain
2171  * cacheline alignment of the DMA). It must be a power of 2.
2172  *
2173  * Various parts of the networking layer expect at least 32 bytes of
2174  * headroom, you should not reduce this.
2175  *
2176  * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
2177  * to reduce average number of cache lines per packet.
2178  * get_rps_cpus() for example only access one 64 bytes aligned block :
2179  * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
2180  */
2181 #ifndef NET_SKB_PAD
2182 #define NET_SKB_PAD	max(32, L1_CACHE_BYTES)
2183 #endif
2184 
2185 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
2186 
__skb_trim(struct sk_buff * skb,unsigned int len)2187 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
2188 {
2189 	if (unlikely(skb_is_nonlinear(skb))) {
2190 		WARN_ON(1);
2191 		return;
2192 	}
2193 	skb->len = len;
2194 	skb_set_tail_pointer(skb, len);
2195 }
2196 
2197 void skb_trim(struct sk_buff *skb, unsigned int len);
2198 
__pskb_trim(struct sk_buff * skb,unsigned int len)2199 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
2200 {
2201 	if (skb->data_len)
2202 		return ___pskb_trim(skb, len);
2203 	__skb_trim(skb, len);
2204 	return 0;
2205 }
2206 
pskb_trim(struct sk_buff * skb,unsigned int len)2207 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
2208 {
2209 	return (len < skb->len) ? __pskb_trim(skb, len) : 0;
2210 }
2211 
2212 /**
2213  *	pskb_trim_unique - remove end from a paged unique (not cloned) buffer
2214  *	@skb: buffer to alter
2215  *	@len: new length
2216  *
2217  *	This is identical to pskb_trim except that the caller knows that
2218  *	the skb is not cloned so we should never get an error due to out-
2219  *	of-memory.
2220  */
pskb_trim_unique(struct sk_buff * skb,unsigned int len)2221 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
2222 {
2223 	int err = pskb_trim(skb, len);
2224 	BUG_ON(err);
2225 }
2226 
2227 /**
2228  *	skb_orphan - orphan a buffer
2229  *	@skb: buffer to orphan
2230  *
2231  *	If a buffer currently has an owner then we call the owner's
2232  *	destructor function and make the @skb unowned. The buffer continues
2233  *	to exist but is no longer charged to its former owner.
2234  */
skb_orphan(struct sk_buff * skb)2235 static inline void skb_orphan(struct sk_buff *skb)
2236 {
2237 	if (skb->destructor) {
2238 		skb->destructor(skb);
2239 		skb->destructor = NULL;
2240 		skb->sk		= NULL;
2241 	} else {
2242 		BUG_ON(skb->sk);
2243 	}
2244 }
2245 
2246 /**
2247  *	skb_orphan_frags - orphan the frags contained in a buffer
2248  *	@skb: buffer to orphan frags from
2249  *	@gfp_mask: allocation mask for replacement pages
2250  *
2251  *	For each frag in the SKB which needs a destructor (i.e. has an
2252  *	owner) create a copy of that frag and release the original
2253  *	page by calling the destructor.
2254  */
skb_orphan_frags(struct sk_buff * skb,gfp_t gfp_mask)2255 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
2256 {
2257 	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY)))
2258 		return 0;
2259 	return skb_copy_ubufs(skb, gfp_mask);
2260 }
2261 
2262 /**
2263  *	__skb_queue_purge - empty a list
2264  *	@list: list to empty
2265  *
2266  *	Delete all buffers on an &sk_buff list. Each buffer is removed from
2267  *	the list and one reference dropped. This function does not take the
2268  *	list lock and the caller must hold the relevant locks to use it.
2269  */
2270 void skb_queue_purge(struct sk_buff_head *list);
__skb_queue_purge(struct sk_buff_head * list)2271 static inline void __skb_queue_purge(struct sk_buff_head *list)
2272 {
2273 	struct sk_buff *skb;
2274 	while ((skb = __skb_dequeue(list)) != NULL)
2275 		kfree_skb(skb);
2276 }
2277 
2278 void *netdev_alloc_frag(unsigned int fragsz);
2279 
2280 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
2281 				   gfp_t gfp_mask);
2282 
2283 /**
2284  *	netdev_alloc_skb - allocate an skbuff for rx on a specific device
2285  *	@dev: network device to receive on
2286  *	@length: length to allocate
2287  *
2288  *	Allocate a new &sk_buff and assign it a usage count of one. The
2289  *	buffer has unspecified headroom built in. Users should allocate
2290  *	the headroom they think they need without accounting for the
2291  *	built in space. The built in space is used for optimisations.
2292  *
2293  *	%NULL is returned if there is no free memory. Although this function
2294  *	allocates memory it can be called from an interrupt.
2295  */
netdev_alloc_skb(struct net_device * dev,unsigned int length)2296 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
2297 					       unsigned int length)
2298 {
2299 	return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
2300 }
2301 
2302 /* legacy helper around __netdev_alloc_skb() */
__dev_alloc_skb(unsigned int length,gfp_t gfp_mask)2303 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
2304 					      gfp_t gfp_mask)
2305 {
2306 	return __netdev_alloc_skb(NULL, length, gfp_mask);
2307 }
2308 
2309 /* legacy helper around netdev_alloc_skb() */
dev_alloc_skb(unsigned int length)2310 static inline struct sk_buff *dev_alloc_skb(unsigned int length)
2311 {
2312 	return netdev_alloc_skb(NULL, length);
2313 }
2314 
2315 
__netdev_alloc_skb_ip_align(struct net_device * dev,unsigned int length,gfp_t gfp)2316 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
2317 		unsigned int length, gfp_t gfp)
2318 {
2319 	struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
2320 
2321 	if (NET_IP_ALIGN && skb)
2322 		skb_reserve(skb, NET_IP_ALIGN);
2323 	return skb;
2324 }
2325 
netdev_alloc_skb_ip_align(struct net_device * dev,unsigned int length)2326 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
2327 		unsigned int length)
2328 {
2329 	return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
2330 }
2331 
skb_free_frag(void * addr)2332 static inline void skb_free_frag(void *addr)
2333 {
2334 	__free_page_frag(addr);
2335 }
2336 
2337 void *napi_alloc_frag(unsigned int fragsz);
2338 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
2339 				 unsigned int length, gfp_t gfp_mask);
napi_alloc_skb(struct napi_struct * napi,unsigned int length)2340 static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,
2341 					     unsigned int length)
2342 {
2343 	return __napi_alloc_skb(napi, length, GFP_ATOMIC);
2344 }
2345 
2346 /**
2347  * __dev_alloc_pages - allocate page for network Rx
2348  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2349  * @order: size of the allocation
2350  *
2351  * Allocate a new page.
2352  *
2353  * %NULL is returned if there is no free memory.
2354 */
__dev_alloc_pages(gfp_t gfp_mask,unsigned int order)2355 static inline struct page *__dev_alloc_pages(gfp_t gfp_mask,
2356 					     unsigned int order)
2357 {
2358 	/* This piece of code contains several assumptions.
2359 	 * 1.  This is for device Rx, therefor a cold page is preferred.
2360 	 * 2.  The expectation is the user wants a compound page.
2361 	 * 3.  If requesting a order 0 page it will not be compound
2362 	 *     due to the check to see if order has a value in prep_new_page
2363 	 * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
2364 	 *     code in gfp_to_alloc_flags that should be enforcing this.
2365 	 */
2366 	gfp_mask |= __GFP_COLD | __GFP_COMP | __GFP_MEMALLOC;
2367 
2368 	return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
2369 }
2370 
dev_alloc_pages(unsigned int order)2371 static inline struct page *dev_alloc_pages(unsigned int order)
2372 {
2373 	return __dev_alloc_pages(GFP_ATOMIC, order);
2374 }
2375 
2376 /**
2377  * __dev_alloc_page - allocate a page for network Rx
2378  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2379  *
2380  * Allocate a new page.
2381  *
2382  * %NULL is returned if there is no free memory.
2383  */
__dev_alloc_page(gfp_t gfp_mask)2384 static inline struct page *__dev_alloc_page(gfp_t gfp_mask)
2385 {
2386 	return __dev_alloc_pages(gfp_mask, 0);
2387 }
2388 
dev_alloc_page(void)2389 static inline struct page *dev_alloc_page(void)
2390 {
2391 	return __dev_alloc_page(GFP_ATOMIC);
2392 }
2393 
2394 /**
2395  *	skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
2396  *	@page: The page that was allocated from skb_alloc_page
2397  *	@skb: The skb that may need pfmemalloc set
2398  */
skb_propagate_pfmemalloc(struct page * page,struct sk_buff * skb)2399 static inline void skb_propagate_pfmemalloc(struct page *page,
2400 					     struct sk_buff *skb)
2401 {
2402 	if (page_is_pfmemalloc(page))
2403 		skb->pfmemalloc = true;
2404 }
2405 
2406 /**
2407  * skb_frag_page - retrieve the page referred to by a paged fragment
2408  * @frag: the paged fragment
2409  *
2410  * Returns the &struct page associated with @frag.
2411  */
skb_frag_page(const skb_frag_t * frag)2412 static inline struct page *skb_frag_page(const skb_frag_t *frag)
2413 {
2414 	return frag->page.p;
2415 }
2416 
2417 /**
2418  * __skb_frag_ref - take an addition reference on a paged fragment.
2419  * @frag: the paged fragment
2420  *
2421  * Takes an additional reference on the paged fragment @frag.
2422  */
__skb_frag_ref(skb_frag_t * frag)2423 static inline void __skb_frag_ref(skb_frag_t *frag)
2424 {
2425 	get_page(skb_frag_page(frag));
2426 }
2427 
2428 /**
2429  * skb_frag_ref - take an addition reference on a paged fragment of an skb.
2430  * @skb: the buffer
2431  * @f: the fragment offset.
2432  *
2433  * Takes an additional reference on the @f'th paged fragment of @skb.
2434  */
skb_frag_ref(struct sk_buff * skb,int f)2435 static inline void skb_frag_ref(struct sk_buff *skb, int f)
2436 {
2437 	__skb_frag_ref(&skb_shinfo(skb)->frags[f]);
2438 }
2439 
2440 /**
2441  * __skb_frag_unref - release a reference on a paged fragment.
2442  * @frag: the paged fragment
2443  *
2444  * Releases a reference on the paged fragment @frag.
2445  */
__skb_frag_unref(skb_frag_t * frag)2446 static inline void __skb_frag_unref(skb_frag_t *frag)
2447 {
2448 	put_page(skb_frag_page(frag));
2449 }
2450 
2451 /**
2452  * skb_frag_unref - release a reference on a paged fragment of an skb.
2453  * @skb: the buffer
2454  * @f: the fragment offset
2455  *
2456  * Releases a reference on the @f'th paged fragment of @skb.
2457  */
skb_frag_unref(struct sk_buff * skb,int f)2458 static inline void skb_frag_unref(struct sk_buff *skb, int f)
2459 {
2460 	__skb_frag_unref(&skb_shinfo(skb)->frags[f]);
2461 }
2462 
2463 /**
2464  * skb_frag_address - gets the address of the data contained in a paged fragment
2465  * @frag: the paged fragment buffer
2466  *
2467  * Returns the address of the data within @frag. The page must already
2468  * be mapped.
2469  */
skb_frag_address(const skb_frag_t * frag)2470 static inline void *skb_frag_address(const skb_frag_t *frag)
2471 {
2472 	return page_address(skb_frag_page(frag)) + frag->page_offset;
2473 }
2474 
2475 /**
2476  * skb_frag_address_safe - gets the address of the data contained in a paged fragment
2477  * @frag: the paged fragment buffer
2478  *
2479  * Returns the address of the data within @frag. Checks that the page
2480  * is mapped and returns %NULL otherwise.
2481  */
skb_frag_address_safe(const skb_frag_t * frag)2482 static inline void *skb_frag_address_safe(const skb_frag_t *frag)
2483 {
2484 	void *ptr = page_address(skb_frag_page(frag));
2485 	if (unlikely(!ptr))
2486 		return NULL;
2487 
2488 	return ptr + frag->page_offset;
2489 }
2490 
2491 /**
2492  * __skb_frag_set_page - sets the page contained in a paged fragment
2493  * @frag: the paged fragment
2494  * @page: the page to set
2495  *
2496  * Sets the fragment @frag to contain @page.
2497  */
__skb_frag_set_page(skb_frag_t * frag,struct page * page)2498 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
2499 {
2500 	frag->page.p = page;
2501 }
2502 
2503 /**
2504  * skb_frag_set_page - sets the page contained in a paged fragment of an skb
2505  * @skb: the buffer
2506  * @f: the fragment offset
2507  * @page: the page to set
2508  *
2509  * Sets the @f'th fragment of @skb to contain @page.
2510  */
skb_frag_set_page(struct sk_buff * skb,int f,struct page * page)2511 static inline void skb_frag_set_page(struct sk_buff *skb, int f,
2512 				     struct page *page)
2513 {
2514 	__skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
2515 }
2516 
2517 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
2518 
2519 /**
2520  * skb_frag_dma_map - maps a paged fragment via the DMA API
2521  * @dev: the device to map the fragment to
2522  * @frag: the paged fragment to map
2523  * @offset: the offset within the fragment (starting at the
2524  *          fragment's own offset)
2525  * @size: the number of bytes to map
2526  * @dir: the direction of the mapping (%PCI_DMA_*)
2527  *
2528  * Maps the page associated with @frag to @device.
2529  */
skb_frag_dma_map(struct device * dev,const skb_frag_t * frag,size_t offset,size_t size,enum dma_data_direction dir)2530 static inline dma_addr_t skb_frag_dma_map(struct device *dev,
2531 					  const skb_frag_t *frag,
2532 					  size_t offset, size_t size,
2533 					  enum dma_data_direction dir)
2534 {
2535 	return dma_map_page(dev, skb_frag_page(frag),
2536 			    frag->page_offset + offset, size, dir);
2537 }
2538 
pskb_copy(struct sk_buff * skb,gfp_t gfp_mask)2539 static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
2540 					gfp_t gfp_mask)
2541 {
2542 	return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
2543 }
2544 
2545 
pskb_copy_for_clone(struct sk_buff * skb,gfp_t gfp_mask)2546 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
2547 						  gfp_t gfp_mask)
2548 {
2549 	return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
2550 }
2551 
2552 
2553 /**
2554  *	skb_clone_writable - is the header of a clone writable
2555  *	@skb: buffer to check
2556  *	@len: length up to which to write
2557  *
2558  *	Returns true if modifying the header part of the cloned buffer
2559  *	does not requires the data to be copied.
2560  */
skb_clone_writable(const struct sk_buff * skb,unsigned int len)2561 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
2562 {
2563 	return !skb_header_cloned(skb) &&
2564 	       skb_headroom(skb) + len <= skb->hdr_len;
2565 }
2566 
__skb_cow(struct sk_buff * skb,unsigned int headroom,int cloned)2567 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
2568 			    int cloned)
2569 {
2570 	int delta = 0;
2571 
2572 	if (headroom > skb_headroom(skb))
2573 		delta = headroom - skb_headroom(skb);
2574 
2575 	if (delta || cloned)
2576 		return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
2577 					GFP_ATOMIC);
2578 	return 0;
2579 }
2580 
2581 /**
2582  *	skb_cow - copy header of skb when it is required
2583  *	@skb: buffer to cow
2584  *	@headroom: needed headroom
2585  *
2586  *	If the skb passed lacks sufficient headroom or its data part
2587  *	is shared, data is reallocated. If reallocation fails, an error
2588  *	is returned and original skb is not changed.
2589  *
2590  *	The result is skb with writable area skb->head...skb->tail
2591  *	and at least @headroom of space at head.
2592  */
skb_cow(struct sk_buff * skb,unsigned int headroom)2593 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
2594 {
2595 	return __skb_cow(skb, headroom, skb_cloned(skb));
2596 }
2597 
2598 /**
2599  *	skb_cow_head - skb_cow but only making the head writable
2600  *	@skb: buffer to cow
2601  *	@headroom: needed headroom
2602  *
2603  *	This function is identical to skb_cow except that we replace the
2604  *	skb_cloned check by skb_header_cloned.  It should be used when
2605  *	you only need to push on some header and do not need to modify
2606  *	the data.
2607  */
skb_cow_head(struct sk_buff * skb,unsigned int headroom)2608 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
2609 {
2610 	return __skb_cow(skb, headroom, skb_header_cloned(skb));
2611 }
2612 
2613 /**
2614  *	skb_padto	- pad an skbuff up to a minimal size
2615  *	@skb: buffer to pad
2616  *	@len: minimal length
2617  *
2618  *	Pads up a buffer to ensure the trailing bytes exist and are
2619  *	blanked. If the buffer already contains sufficient data it
2620  *	is untouched. Otherwise it is extended. Returns zero on
2621  *	success. The skb is freed on error.
2622  */
skb_padto(struct sk_buff * skb,unsigned int len)2623 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
2624 {
2625 	unsigned int size = skb->len;
2626 	if (likely(size >= len))
2627 		return 0;
2628 	return skb_pad(skb, len - size);
2629 }
2630 
2631 /**
2632  *	skb_put_padto - increase size and pad an skbuff up to a minimal size
2633  *	@skb: buffer to pad
2634  *	@len: minimal length
2635  *
2636  *	Pads up a buffer to ensure the trailing bytes exist and are
2637  *	blanked. If the buffer already contains sufficient data it
2638  *	is untouched. Otherwise it is extended. Returns zero on
2639  *	success. The skb is freed on error.
2640  */
skb_put_padto(struct sk_buff * skb,unsigned int len)2641 static inline int skb_put_padto(struct sk_buff *skb, unsigned int len)
2642 {
2643 	unsigned int size = skb->len;
2644 
2645 	if (unlikely(size < len)) {
2646 		len -= size;
2647 		if (skb_pad(skb, len))
2648 			return -ENOMEM;
2649 		__skb_put(skb, len);
2650 	}
2651 	return 0;
2652 }
2653 
skb_add_data(struct sk_buff * skb,struct iov_iter * from,int copy)2654 static inline int skb_add_data(struct sk_buff *skb,
2655 			       struct iov_iter *from, int copy)
2656 {
2657 	const int off = skb->len;
2658 
2659 	if (skb->ip_summed == CHECKSUM_NONE) {
2660 		__wsum csum = 0;
2661 		if (csum_and_copy_from_iter(skb_put(skb, copy), copy,
2662 					    &csum, from) == copy) {
2663 			skb->csum = csum_block_add(skb->csum, csum, off);
2664 			return 0;
2665 		}
2666 	} else if (copy_from_iter(skb_put(skb, copy), copy, from) == copy)
2667 		return 0;
2668 
2669 	__skb_trim(skb, off);
2670 	return -EFAULT;
2671 }
2672 
skb_can_coalesce(struct sk_buff * skb,int i,const struct page * page,int off)2673 static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
2674 				    const struct page *page, int off)
2675 {
2676 	if (i) {
2677 		const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
2678 
2679 		return page == skb_frag_page(frag) &&
2680 		       off == frag->page_offset + skb_frag_size(frag);
2681 	}
2682 	return false;
2683 }
2684 
__skb_linearize(struct sk_buff * skb)2685 static inline int __skb_linearize(struct sk_buff *skb)
2686 {
2687 	return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
2688 }
2689 
2690 /**
2691  *	skb_linearize - convert paged skb to linear one
2692  *	@skb: buffer to linarize
2693  *
2694  *	If there is no free memory -ENOMEM is returned, otherwise zero
2695  *	is returned and the old skb data released.
2696  */
skb_linearize(struct sk_buff * skb)2697 static inline int skb_linearize(struct sk_buff *skb)
2698 {
2699 	return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
2700 }
2701 
2702 /**
2703  * skb_has_shared_frag - can any frag be overwritten
2704  * @skb: buffer to test
2705  *
2706  * Return true if the skb has at least one frag that might be modified
2707  * by an external entity (as in vmsplice()/sendfile())
2708  */
skb_has_shared_frag(const struct sk_buff * skb)2709 static inline bool skb_has_shared_frag(const struct sk_buff *skb)
2710 {
2711 	return skb_is_nonlinear(skb) &&
2712 	       skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
2713 }
2714 
2715 /**
2716  *	skb_linearize_cow - make sure skb is linear and writable
2717  *	@skb: buffer to process
2718  *
2719  *	If there is no free memory -ENOMEM is returned, otherwise zero
2720  *	is returned and the old skb data released.
2721  */
skb_linearize_cow(struct sk_buff * skb)2722 static inline int skb_linearize_cow(struct sk_buff *skb)
2723 {
2724 	return skb_is_nonlinear(skb) || skb_cloned(skb) ?
2725 	       __skb_linearize(skb) : 0;
2726 }
2727 
2728 /**
2729  *	skb_postpull_rcsum - update checksum for received skb after pull
2730  *	@skb: buffer to update
2731  *	@start: start of data before pull
2732  *	@len: length of data pulled
2733  *
2734  *	After doing a pull on a received packet, you need to call this to
2735  *	update the CHECKSUM_COMPLETE checksum, or set ip_summed to
2736  *	CHECKSUM_NONE so that it can be recomputed from scratch.
2737  */
2738 
skb_postpull_rcsum(struct sk_buff * skb,const void * start,unsigned int len)2739 static inline void skb_postpull_rcsum(struct sk_buff *skb,
2740 				      const void *start, unsigned int len)
2741 {
2742 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2743 		skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
2744 	else if (skb->ip_summed == CHECKSUM_PARTIAL &&
2745 		 skb_checksum_start_offset(skb) < 0)
2746 		skb->ip_summed = CHECKSUM_NONE;
2747 }
2748 
2749 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
2750 
skb_postpush_rcsum(struct sk_buff * skb,const void * start,unsigned int len)2751 static inline void skb_postpush_rcsum(struct sk_buff *skb,
2752 				      const void *start, unsigned int len)
2753 {
2754 	/* For performing the reverse operation to skb_postpull_rcsum(),
2755 	 * we can instead of ...
2756 	 *
2757 	 *   skb->csum = csum_add(skb->csum, csum_partial(start, len, 0));
2758 	 *
2759 	 * ... just use this equivalent version here to save a few
2760 	 * instructions. Feeding csum of 0 in csum_partial() and later
2761 	 * on adding skb->csum is equivalent to feed skb->csum in the
2762 	 * first place.
2763 	 */
2764 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2765 		skb->csum = csum_partial(start, len, skb->csum);
2766 }
2767 
2768 /**
2769  *	pskb_trim_rcsum - trim received skb and update checksum
2770  *	@skb: buffer to trim
2771  *	@len: new length
2772  *
2773  *	This is exactly the same as pskb_trim except that it ensures the
2774  *	checksum of received packets are still valid after the operation.
2775  */
2776 
pskb_trim_rcsum(struct sk_buff * skb,unsigned int len)2777 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
2778 {
2779 	if (likely(len >= skb->len))
2780 		return 0;
2781 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2782 		skb->ip_summed = CHECKSUM_NONE;
2783 	return __pskb_trim(skb, len);
2784 }
2785 
2786 #define skb_queue_walk(queue, skb) \
2787 		for (skb = (queue)->next;					\
2788 		     skb != (struct sk_buff *)(queue);				\
2789 		     skb = skb->next)
2790 
2791 #define skb_queue_walk_safe(queue, skb, tmp)					\
2792 		for (skb = (queue)->next, tmp = skb->next;			\
2793 		     skb != (struct sk_buff *)(queue);				\
2794 		     skb = tmp, tmp = skb->next)
2795 
2796 #define skb_queue_walk_from(queue, skb)						\
2797 		for (; skb != (struct sk_buff *)(queue);			\
2798 		     skb = skb->next)
2799 
2800 #define skb_queue_walk_from_safe(queue, skb, tmp)				\
2801 		for (tmp = skb->next;						\
2802 		     skb != (struct sk_buff *)(queue);				\
2803 		     skb = tmp, tmp = skb->next)
2804 
2805 #define skb_queue_reverse_walk(queue, skb) \
2806 		for (skb = (queue)->prev;					\
2807 		     skb != (struct sk_buff *)(queue);				\
2808 		     skb = skb->prev)
2809 
2810 #define skb_queue_reverse_walk_safe(queue, skb, tmp)				\
2811 		for (skb = (queue)->prev, tmp = skb->prev;			\
2812 		     skb != (struct sk_buff *)(queue);				\
2813 		     skb = tmp, tmp = skb->prev)
2814 
2815 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp)			\
2816 		for (tmp = skb->prev;						\
2817 		     skb != (struct sk_buff *)(queue);				\
2818 		     skb = tmp, tmp = skb->prev)
2819 
skb_has_frag_list(const struct sk_buff * skb)2820 static inline bool skb_has_frag_list(const struct sk_buff *skb)
2821 {
2822 	return skb_shinfo(skb)->frag_list != NULL;
2823 }
2824 
skb_frag_list_init(struct sk_buff * skb)2825 static inline void skb_frag_list_init(struct sk_buff *skb)
2826 {
2827 	skb_shinfo(skb)->frag_list = NULL;
2828 }
2829 
2830 #define skb_walk_frags(skb, iter)	\
2831 	for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
2832 
2833 struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
2834 				    int *peeked, int *off, int *err);
2835 struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, int noblock,
2836 				  int *err);
2837 unsigned int datagram_poll(struct file *file, struct socket *sock,
2838 			   struct poll_table_struct *wait);
2839 int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
2840 			   struct iov_iter *to, int size);
skb_copy_datagram_msg(const struct sk_buff * from,int offset,struct msghdr * msg,int size)2841 static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
2842 					struct msghdr *msg, int size)
2843 {
2844 	return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
2845 }
2846 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
2847 				   struct msghdr *msg);
2848 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
2849 				 struct iov_iter *from, int len);
2850 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
2851 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
2852 void skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb);
2853 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
2854 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
2855 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
2856 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
2857 			      int len, __wsum csum);
2858 ssize_t skb_socket_splice(struct sock *sk,
2859 			  struct pipe_inode_info *pipe,
2860 			  struct splice_pipe_desc *spd);
2861 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2862 		    struct pipe_inode_info *pipe, unsigned int len,
2863 		    unsigned int flags,
2864 		    ssize_t (*splice_cb)(struct sock *,
2865 					 struct pipe_inode_info *,
2866 					 struct splice_pipe_desc *));
2867 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
2868 unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
2869 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
2870 		 int len, int hlen);
2871 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
2872 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
2873 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
2874 unsigned int skb_gso_transport_seglen(const struct sk_buff *skb);
2875 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
2876 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
2877 int skb_ensure_writable(struct sk_buff *skb, int write_len);
2878 int skb_vlan_pop(struct sk_buff *skb);
2879 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
2880 
memcpy_from_msg(void * data,struct msghdr * msg,int len)2881 static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
2882 {
2883 	return copy_from_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
2884 }
2885 
memcpy_to_msg(struct msghdr * msg,void * data,int len)2886 static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
2887 {
2888 	return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
2889 }
2890 
2891 struct skb_checksum_ops {
2892 	__wsum (*update)(const void *mem, int len, __wsum wsum);
2893 	__wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
2894 };
2895 
2896 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2897 		      __wsum csum, const struct skb_checksum_ops *ops);
2898 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
2899 		    __wsum csum);
2900 
2901 static inline void * __must_check
__skb_header_pointer(const struct sk_buff * skb,int offset,int len,void * data,int hlen,void * buffer)2902 __skb_header_pointer(const struct sk_buff *skb, int offset,
2903 		     int len, void *data, int hlen, void *buffer)
2904 {
2905 	if (hlen - offset >= len)
2906 		return data + offset;
2907 
2908 	if (!skb ||
2909 	    skb_copy_bits(skb, offset, buffer, len) < 0)
2910 		return NULL;
2911 
2912 	return buffer;
2913 }
2914 
2915 static inline void * __must_check
skb_header_pointer(const struct sk_buff * skb,int offset,int len,void * buffer)2916 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
2917 {
2918 	return __skb_header_pointer(skb, offset, len, skb->data,
2919 				    skb_headlen(skb), buffer);
2920 }
2921 
2922 /**
2923  *	skb_needs_linearize - check if we need to linearize a given skb
2924  *			      depending on the given device features.
2925  *	@skb: socket buffer to check
2926  *	@features: net device features
2927  *
2928  *	Returns true if either:
2929  *	1. skb has frag_list and the device doesn't support FRAGLIST, or
2930  *	2. skb is fragmented and the device does not support SG.
2931  */
skb_needs_linearize(struct sk_buff * skb,netdev_features_t features)2932 static inline bool skb_needs_linearize(struct sk_buff *skb,
2933 				       netdev_features_t features)
2934 {
2935 	return skb_is_nonlinear(skb) &&
2936 	       ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
2937 		(skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
2938 }
2939 
skb_copy_from_linear_data(const struct sk_buff * skb,void * to,const unsigned int len)2940 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
2941 					     void *to,
2942 					     const unsigned int len)
2943 {
2944 	memcpy(to, skb->data, len);
2945 }
2946 
skb_copy_from_linear_data_offset(const struct sk_buff * skb,const int offset,void * to,const unsigned int len)2947 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
2948 						    const int offset, void *to,
2949 						    const unsigned int len)
2950 {
2951 	memcpy(to, skb->data + offset, len);
2952 }
2953 
skb_copy_to_linear_data(struct sk_buff * skb,const void * from,const unsigned int len)2954 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
2955 					   const void *from,
2956 					   const unsigned int len)
2957 {
2958 	memcpy(skb->data, from, len);
2959 }
2960 
skb_copy_to_linear_data_offset(struct sk_buff * skb,const int offset,const void * from,const unsigned int len)2961 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
2962 						  const int offset,
2963 						  const void *from,
2964 						  const unsigned int len)
2965 {
2966 	memcpy(skb->data + offset, from, len);
2967 }
2968 
2969 void skb_init(void);
2970 
skb_get_ktime(const struct sk_buff * skb)2971 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
2972 {
2973 	return skb->tstamp;
2974 }
2975 
2976 /**
2977  *	skb_get_timestamp - get timestamp from a skb
2978  *	@skb: skb to get stamp from
2979  *	@stamp: pointer to struct timeval to store stamp in
2980  *
2981  *	Timestamps are stored in the skb as offsets to a base timestamp.
2982  *	This function converts the offset back to a struct timeval and stores
2983  *	it in stamp.
2984  */
skb_get_timestamp(const struct sk_buff * skb,struct timeval * stamp)2985 static inline void skb_get_timestamp(const struct sk_buff *skb,
2986 				     struct timeval *stamp)
2987 {
2988 	*stamp = ktime_to_timeval(skb->tstamp);
2989 }
2990 
skb_get_timestampns(const struct sk_buff * skb,struct timespec * stamp)2991 static inline void skb_get_timestampns(const struct sk_buff *skb,
2992 				       struct timespec *stamp)
2993 {
2994 	*stamp = ktime_to_timespec(skb->tstamp);
2995 }
2996 
__net_timestamp(struct sk_buff * skb)2997 static inline void __net_timestamp(struct sk_buff *skb)
2998 {
2999 	skb->tstamp = ktime_get_real();
3000 }
3001 
net_timedelta(ktime_t t)3002 static inline ktime_t net_timedelta(ktime_t t)
3003 {
3004 	return ktime_sub(ktime_get_real(), t);
3005 }
3006 
net_invalid_timestamp(void)3007 static inline ktime_t net_invalid_timestamp(void)
3008 {
3009 	return ktime_set(0, 0);
3010 }
3011 
3012 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
3013 
3014 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
3015 
3016 void skb_clone_tx_timestamp(struct sk_buff *skb);
3017 bool skb_defer_rx_timestamp(struct sk_buff *skb);
3018 
3019 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
3020 
skb_clone_tx_timestamp(struct sk_buff * skb)3021 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
3022 {
3023 }
3024 
skb_defer_rx_timestamp(struct sk_buff * skb)3025 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
3026 {
3027 	return false;
3028 }
3029 
3030 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
3031 
3032 /**
3033  * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
3034  *
3035  * PHY drivers may accept clones of transmitted packets for
3036  * timestamping via their phy_driver.txtstamp method. These drivers
3037  * must call this function to return the skb back to the stack with a
3038  * timestamp.
3039  *
3040  * @skb: clone of the the original outgoing packet
3041  * @hwtstamps: hardware time stamps
3042  *
3043  */
3044 void skb_complete_tx_timestamp(struct sk_buff *skb,
3045 			       struct skb_shared_hwtstamps *hwtstamps);
3046 
3047 void __skb_tstamp_tx(struct sk_buff *orig_skb,
3048 		     struct skb_shared_hwtstamps *hwtstamps,
3049 		     struct sock *sk, int tstype);
3050 
3051 /**
3052  * skb_tstamp_tx - queue clone of skb with send time stamps
3053  * @orig_skb:	the original outgoing packet
3054  * @hwtstamps:	hardware time stamps, may be NULL if not available
3055  *
3056  * If the skb has a socket associated, then this function clones the
3057  * skb (thus sharing the actual data and optional structures), stores
3058  * the optional hardware time stamping information (if non NULL) or
3059  * generates a software time stamp (otherwise), then queues the clone
3060  * to the error queue of the socket.  Errors are silently ignored.
3061  */
3062 void skb_tstamp_tx(struct sk_buff *orig_skb,
3063 		   struct skb_shared_hwtstamps *hwtstamps);
3064 
sw_tx_timestamp(struct sk_buff * skb)3065 static inline void sw_tx_timestamp(struct sk_buff *skb)
3066 {
3067 	if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP &&
3068 	    !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
3069 		skb_tstamp_tx(skb, NULL);
3070 }
3071 
3072 /**
3073  * skb_tx_timestamp() - Driver hook for transmit timestamping
3074  *
3075  * Ethernet MAC Drivers should call this function in their hard_xmit()
3076  * function immediately before giving the sk_buff to the MAC hardware.
3077  *
3078  * Specifically, one should make absolutely sure that this function is
3079  * called before TX completion of this packet can trigger.  Otherwise
3080  * the packet could potentially already be freed.
3081  *
3082  * @skb: A socket buffer.
3083  */
skb_tx_timestamp(struct sk_buff * skb)3084 static inline void skb_tx_timestamp(struct sk_buff *skb)
3085 {
3086 	skb_clone_tx_timestamp(skb);
3087 	sw_tx_timestamp(skb);
3088 }
3089 
3090 /**
3091  * skb_complete_wifi_ack - deliver skb with wifi status
3092  *
3093  * @skb: the original outgoing packet
3094  * @acked: ack status
3095  *
3096  */
3097 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
3098 
3099 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
3100 __sum16 __skb_checksum_complete(struct sk_buff *skb);
3101 
skb_csum_unnecessary(const struct sk_buff * skb)3102 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
3103 {
3104 	return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
3105 		skb->csum_valid ||
3106 		(skb->ip_summed == CHECKSUM_PARTIAL &&
3107 		 skb_checksum_start_offset(skb) >= 0));
3108 }
3109 
3110 /**
3111  *	skb_checksum_complete - Calculate checksum of an entire packet
3112  *	@skb: packet to process
3113  *
3114  *	This function calculates the checksum over the entire packet plus
3115  *	the value of skb->csum.  The latter can be used to supply the
3116  *	checksum of a pseudo header as used by TCP/UDP.  It returns the
3117  *	checksum.
3118  *
3119  *	For protocols that contain complete checksums such as ICMP/TCP/UDP,
3120  *	this function can be used to verify that checksum on received
3121  *	packets.  In that case the function should return zero if the
3122  *	checksum is correct.  In particular, this function will return zero
3123  *	if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
3124  *	hardware has already verified the correctness of the checksum.
3125  */
skb_checksum_complete(struct sk_buff * skb)3126 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
3127 {
3128 	return skb_csum_unnecessary(skb) ?
3129 	       0 : __skb_checksum_complete(skb);
3130 }
3131 
__skb_decr_checksum_unnecessary(struct sk_buff * skb)3132 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
3133 {
3134 	if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3135 		if (skb->csum_level == 0)
3136 			skb->ip_summed = CHECKSUM_NONE;
3137 		else
3138 			skb->csum_level--;
3139 	}
3140 }
3141 
__skb_incr_checksum_unnecessary(struct sk_buff * skb)3142 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
3143 {
3144 	if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3145 		if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
3146 			skb->csum_level++;
3147 	} else if (skb->ip_summed == CHECKSUM_NONE) {
3148 		skb->ip_summed = CHECKSUM_UNNECESSARY;
3149 		skb->csum_level = 0;
3150 	}
3151 }
3152 
__skb_mark_checksum_bad(struct sk_buff * skb)3153 static inline void __skb_mark_checksum_bad(struct sk_buff *skb)
3154 {
3155 	/* Mark current checksum as bad (typically called from GRO
3156 	 * path). In the case that ip_summed is CHECKSUM_NONE
3157 	 * this must be the first checksum encountered in the packet.
3158 	 * When ip_summed is CHECKSUM_UNNECESSARY, this is the first
3159 	 * checksum after the last one validated. For UDP, a zero
3160 	 * checksum can not be marked as bad.
3161 	 */
3162 
3163 	if (skb->ip_summed == CHECKSUM_NONE ||
3164 	    skb->ip_summed == CHECKSUM_UNNECESSARY)
3165 		skb->csum_bad = 1;
3166 }
3167 
3168 /* Check if we need to perform checksum complete validation.
3169  *
3170  * Returns true if checksum complete is needed, false otherwise
3171  * (either checksum is unnecessary or zero checksum is allowed).
3172  */
__skb_checksum_validate_needed(struct sk_buff * skb,bool zero_okay,__sum16 check)3173 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
3174 						  bool zero_okay,
3175 						  __sum16 check)
3176 {
3177 	if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
3178 		skb->csum_valid = 1;
3179 		__skb_decr_checksum_unnecessary(skb);
3180 		return false;
3181 	}
3182 
3183 	return true;
3184 }
3185 
3186 /* For small packets <= CHECKSUM_BREAK peform checksum complete directly
3187  * in checksum_init.
3188  */
3189 #define CHECKSUM_BREAK 76
3190 
3191 /* Unset checksum-complete
3192  *
3193  * Unset checksum complete can be done when packet is being modified
3194  * (uncompressed for instance) and checksum-complete value is
3195  * invalidated.
3196  */
skb_checksum_complete_unset(struct sk_buff * skb)3197 static inline void skb_checksum_complete_unset(struct sk_buff *skb)
3198 {
3199 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3200 		skb->ip_summed = CHECKSUM_NONE;
3201 }
3202 
3203 /* Validate (init) checksum based on checksum complete.
3204  *
3205  * Return values:
3206  *   0: checksum is validated or try to in skb_checksum_complete. In the latter
3207  *	case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
3208  *	checksum is stored in skb->csum for use in __skb_checksum_complete
3209  *   non-zero: value of invalid checksum
3210  *
3211  */
__skb_checksum_validate_complete(struct sk_buff * skb,bool complete,__wsum psum)3212 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
3213 						       bool complete,
3214 						       __wsum psum)
3215 {
3216 	if (skb->ip_summed == CHECKSUM_COMPLETE) {
3217 		if (!csum_fold(csum_add(psum, skb->csum))) {
3218 			skb->csum_valid = 1;
3219 			return 0;
3220 		}
3221 	} else if (skb->csum_bad) {
3222 		/* ip_summed == CHECKSUM_NONE in this case */
3223 		return (__force __sum16)1;
3224 	}
3225 
3226 	skb->csum = psum;
3227 
3228 	if (complete || skb->len <= CHECKSUM_BREAK) {
3229 		__sum16 csum;
3230 
3231 		csum = __skb_checksum_complete(skb);
3232 		skb->csum_valid = !csum;
3233 		return csum;
3234 	}
3235 
3236 	return 0;
3237 }
3238 
null_compute_pseudo(struct sk_buff * skb,int proto)3239 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
3240 {
3241 	return 0;
3242 }
3243 
3244 /* Perform checksum validate (init). Note that this is a macro since we only
3245  * want to calculate the pseudo header which is an input function if necessary.
3246  * First we try to validate without any computation (checksum unnecessary) and
3247  * then calculate based on checksum complete calling the function to compute
3248  * pseudo header.
3249  *
3250  * Return values:
3251  *   0: checksum is validated or try to in skb_checksum_complete
3252  *   non-zero: value of invalid checksum
3253  */
3254 #define __skb_checksum_validate(skb, proto, complete,			\
3255 				zero_okay, check, compute_pseudo)	\
3256 ({									\
3257 	__sum16 __ret = 0;						\
3258 	skb->csum_valid = 0;						\
3259 	if (__skb_checksum_validate_needed(skb, zero_okay, check))	\
3260 		__ret = __skb_checksum_validate_complete(skb,		\
3261 				complete, compute_pseudo(skb, proto));	\
3262 	__ret;								\
3263 })
3264 
3265 #define skb_checksum_init(skb, proto, compute_pseudo)			\
3266 	__skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
3267 
3268 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo)	\
3269 	__skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
3270 
3271 #define skb_checksum_validate(skb, proto, compute_pseudo)		\
3272 	__skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
3273 
3274 #define skb_checksum_validate_zero_check(skb, proto, check,		\
3275 					 compute_pseudo)		\
3276 	__skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
3277 
3278 #define skb_checksum_simple_validate(skb)				\
3279 	__skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
3280 
__skb_checksum_convert_check(struct sk_buff * skb)3281 static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
3282 {
3283 	return (skb->ip_summed == CHECKSUM_NONE &&
3284 		skb->csum_valid && !skb->csum_bad);
3285 }
3286 
__skb_checksum_convert(struct sk_buff * skb,__sum16 check,__wsum pseudo)3287 static inline void __skb_checksum_convert(struct sk_buff *skb,
3288 					  __sum16 check, __wsum pseudo)
3289 {
3290 	skb->csum = ~pseudo;
3291 	skb->ip_summed = CHECKSUM_COMPLETE;
3292 }
3293 
3294 #define skb_checksum_try_convert(skb, proto, check, compute_pseudo)	\
3295 do {									\
3296 	if (__skb_checksum_convert_check(skb))				\
3297 		__skb_checksum_convert(skb, check,			\
3298 				       compute_pseudo(skb, proto));	\
3299 } while (0)
3300 
skb_remcsum_adjust_partial(struct sk_buff * skb,void * ptr,u16 start,u16 offset)3301 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
3302 					      u16 start, u16 offset)
3303 {
3304 	skb->ip_summed = CHECKSUM_PARTIAL;
3305 	skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
3306 	skb->csum_offset = offset - start;
3307 }
3308 
3309 /* Update skbuf and packet to reflect the remote checksum offload operation.
3310  * When called, ptr indicates the starting point for skb->csum when
3311  * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
3312  * here, skb_postpull_rcsum is done so skb->csum start is ptr.
3313  */
skb_remcsum_process(struct sk_buff * skb,void * ptr,int start,int offset,bool nopartial)3314 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
3315 				       int start, int offset, bool nopartial)
3316 {
3317 	__wsum delta;
3318 
3319 	if (!nopartial) {
3320 		skb_remcsum_adjust_partial(skb, ptr, start, offset);
3321 		return;
3322 	}
3323 
3324 	 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
3325 		__skb_checksum_complete(skb);
3326 		skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
3327 	}
3328 
3329 	delta = remcsum_adjust(ptr, skb->csum, start, offset);
3330 
3331 	/* Adjust skb->csum since we changed the packet */
3332 	skb->csum = csum_add(skb->csum, delta);
3333 }
3334 
3335 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3336 void nf_conntrack_destroy(struct nf_conntrack *nfct);
nf_conntrack_put(struct nf_conntrack * nfct)3337 static inline void nf_conntrack_put(struct nf_conntrack *nfct)
3338 {
3339 	if (nfct && atomic_dec_and_test(&nfct->use))
3340 		nf_conntrack_destroy(nfct);
3341 }
nf_conntrack_get(struct nf_conntrack * nfct)3342 static inline void nf_conntrack_get(struct nf_conntrack *nfct)
3343 {
3344 	if (nfct)
3345 		atomic_inc(&nfct->use);
3346 }
3347 #endif
3348 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
nf_bridge_put(struct nf_bridge_info * nf_bridge)3349 static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge)
3350 {
3351 	if (nf_bridge && atomic_dec_and_test(&nf_bridge->use))
3352 		kfree(nf_bridge);
3353 }
nf_bridge_get(struct nf_bridge_info * nf_bridge)3354 static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
3355 {
3356 	if (nf_bridge)
3357 		atomic_inc(&nf_bridge->use);
3358 }
3359 #endif /* CONFIG_BRIDGE_NETFILTER */
nf_reset(struct sk_buff * skb)3360 static inline void nf_reset(struct sk_buff *skb)
3361 {
3362 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3363 	nf_conntrack_put(skb->nfct);
3364 	skb->nfct = NULL;
3365 #endif
3366 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3367 	nf_bridge_put(skb->nf_bridge);
3368 	skb->nf_bridge = NULL;
3369 #endif
3370 }
3371 
nf_reset_trace(struct sk_buff * skb)3372 static inline void nf_reset_trace(struct sk_buff *skb)
3373 {
3374 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
3375 	skb->nf_trace = 0;
3376 #endif
3377 }
3378 
3379 /* Note: This doesn't put any conntrack and bridge info in dst. */
__nf_copy(struct sk_buff * dst,const struct sk_buff * src,bool copy)3380 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
3381 			     bool copy)
3382 {
3383 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3384 	dst->nfct = src->nfct;
3385 	nf_conntrack_get(src->nfct);
3386 	if (copy)
3387 		dst->nfctinfo = src->nfctinfo;
3388 #endif
3389 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3390 	dst->nf_bridge  = src->nf_bridge;
3391 	nf_bridge_get(src->nf_bridge);
3392 #endif
3393 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
3394 	if (copy)
3395 		dst->nf_trace = src->nf_trace;
3396 #endif
3397 }
3398 
nf_copy(struct sk_buff * dst,const struct sk_buff * src)3399 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
3400 {
3401 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
3402 	nf_conntrack_put(dst->nfct);
3403 #endif
3404 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
3405 	nf_bridge_put(dst->nf_bridge);
3406 #endif
3407 	__nf_copy(dst, src, true);
3408 }
3409 
3410 #ifdef CONFIG_NETWORK_SECMARK
skb_copy_secmark(struct sk_buff * to,const struct sk_buff * from)3411 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
3412 {
3413 	to->secmark = from->secmark;
3414 }
3415 
skb_init_secmark(struct sk_buff * skb)3416 static inline void skb_init_secmark(struct sk_buff *skb)
3417 {
3418 	skb->secmark = 0;
3419 }
3420 #else
skb_copy_secmark(struct sk_buff * to,const struct sk_buff * from)3421 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
3422 { }
3423 
skb_init_secmark(struct sk_buff * skb)3424 static inline void skb_init_secmark(struct sk_buff *skb)
3425 { }
3426 #endif
3427 
skb_irq_freeable(const struct sk_buff * skb)3428 static inline bool skb_irq_freeable(const struct sk_buff *skb)
3429 {
3430 	return !skb->destructor &&
3431 #if IS_ENABLED(CONFIG_XFRM)
3432 		!skb->sp &&
3433 #endif
3434 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
3435 		!skb->nfct &&
3436 #endif
3437 		!skb->_skb_refdst &&
3438 		!skb_has_frag_list(skb);
3439 }
3440 
skb_set_queue_mapping(struct sk_buff * skb,u16 queue_mapping)3441 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
3442 {
3443 	skb->queue_mapping = queue_mapping;
3444 }
3445 
skb_get_queue_mapping(const struct sk_buff * skb)3446 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
3447 {
3448 	return skb->queue_mapping;
3449 }
3450 
skb_copy_queue_mapping(struct sk_buff * to,const struct sk_buff * from)3451 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
3452 {
3453 	to->queue_mapping = from->queue_mapping;
3454 }
3455 
skb_record_rx_queue(struct sk_buff * skb,u16 rx_queue)3456 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
3457 {
3458 	skb->queue_mapping = rx_queue + 1;
3459 }
3460 
skb_get_rx_queue(const struct sk_buff * skb)3461 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
3462 {
3463 	return skb->queue_mapping - 1;
3464 }
3465 
skb_rx_queue_recorded(const struct sk_buff * skb)3466 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
3467 {
3468 	return skb->queue_mapping != 0;
3469 }
3470 
skb_sec_path(struct sk_buff * skb)3471 static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
3472 {
3473 #ifdef CONFIG_XFRM
3474 	return skb->sp;
3475 #else
3476 	return NULL;
3477 #endif
3478 }
3479 
3480 /* Keeps track of mac header offset relative to skb->head.
3481  * It is useful for TSO of Tunneling protocol. e.g. GRE.
3482  * For non-tunnel skb it points to skb_mac_header() and for
3483  * tunnel skb it points to outer mac header.
3484  * Keeps track of level of encapsulation of network headers.
3485  */
3486 struct skb_gso_cb {
3487 	int	mac_offset;
3488 	int	encap_level;
3489 	__u16	csum_start;
3490 };
3491 #define SKB_SGO_CB_OFFSET	32
3492 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_SGO_CB_OFFSET))
3493 
skb_tnl_header_len(const struct sk_buff * inner_skb)3494 static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
3495 {
3496 	return (skb_mac_header(inner_skb) - inner_skb->head) -
3497 		SKB_GSO_CB(inner_skb)->mac_offset;
3498 }
3499 
gso_pskb_expand_head(struct sk_buff * skb,int extra)3500 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
3501 {
3502 	int new_headroom, headroom;
3503 	int ret;
3504 
3505 	headroom = skb_headroom(skb);
3506 	ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
3507 	if (ret)
3508 		return ret;
3509 
3510 	new_headroom = skb_headroom(skb);
3511 	SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
3512 	return 0;
3513 }
3514 
3515 /* Compute the checksum for a gso segment. First compute the checksum value
3516  * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
3517  * then add in skb->csum (checksum from csum_start to end of packet).
3518  * skb->csum and csum_start are then updated to reflect the checksum of the
3519  * resultant packet starting from the transport header-- the resultant checksum
3520  * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo
3521  * header.
3522  */
gso_make_checksum(struct sk_buff * skb,__wsum res)3523 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res)
3524 {
3525 	int plen = SKB_GSO_CB(skb)->csum_start - skb_headroom(skb) -
3526 		   skb_transport_offset(skb);
3527 	__wsum partial;
3528 
3529 	partial = csum_partial(skb_transport_header(skb), plen, skb->csum);
3530 	skb->csum = res;
3531 	SKB_GSO_CB(skb)->csum_start -= plen;
3532 
3533 	return csum_fold(partial);
3534 }
3535 
skb_is_gso(const struct sk_buff * skb)3536 static inline bool skb_is_gso(const struct sk_buff *skb)
3537 {
3538 	return skb_shinfo(skb)->gso_size;
3539 }
3540 
3541 /* Note: Should be called only if skb_is_gso(skb) is true */
skb_is_gso_v6(const struct sk_buff * skb)3542 static inline bool skb_is_gso_v6(const struct sk_buff *skb)
3543 {
3544 	return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
3545 }
3546 
3547 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
3548 
skb_warn_if_lro(const struct sk_buff * skb)3549 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
3550 {
3551 	/* LRO sets gso_size but not gso_type, whereas if GSO is really
3552 	 * wanted then gso_type will be set. */
3553 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
3554 
3555 	if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
3556 	    unlikely(shinfo->gso_type == 0)) {
3557 		__skb_warn_lro_forwarding(skb);
3558 		return true;
3559 	}
3560 	return false;
3561 }
3562 
skb_forward_csum(struct sk_buff * skb)3563 static inline void skb_forward_csum(struct sk_buff *skb)
3564 {
3565 	/* Unfortunately we don't support this one.  Any brave souls? */
3566 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3567 		skb->ip_summed = CHECKSUM_NONE;
3568 }
3569 
3570 /**
3571  * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
3572  * @skb: skb to check
3573  *
3574  * fresh skbs have their ip_summed set to CHECKSUM_NONE.
3575  * Instead of forcing ip_summed to CHECKSUM_NONE, we can
3576  * use this helper, to document places where we make this assertion.
3577  */
skb_checksum_none_assert(const struct sk_buff * skb)3578 static inline void skb_checksum_none_assert(const struct sk_buff *skb)
3579 {
3580 #ifdef DEBUG
3581 	BUG_ON(skb->ip_summed != CHECKSUM_NONE);
3582 #endif
3583 }
3584 
3585 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
3586 
3587 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
3588 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
3589 				     unsigned int transport_len,
3590 				     __sum16(*skb_chkf)(struct sk_buff *skb));
3591 
3592 /**
3593  * skb_head_is_locked - Determine if the skb->head is locked down
3594  * @skb: skb to check
3595  *
3596  * The head on skbs build around a head frag can be removed if they are
3597  * not cloned.  This function returns true if the skb head is locked down
3598  * due to either being allocated via kmalloc, or by being a clone with
3599  * multiple references to the head.
3600  */
skb_head_is_locked(const struct sk_buff * skb)3601 static inline bool skb_head_is_locked(const struct sk_buff *skb)
3602 {
3603 	return !skb->head_frag || skb_cloned(skb);
3604 }
3605 
3606 /**
3607  * skb_gso_network_seglen - Return length of individual segments of a gso packet
3608  *
3609  * @skb: GSO skb
3610  *
3611  * skb_gso_network_seglen is used to determine the real size of the
3612  * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
3613  *
3614  * The MAC/L2 header is not accounted for.
3615  */
skb_gso_network_seglen(const struct sk_buff * skb)3616 static inline unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
3617 {
3618 	unsigned int hdr_len = skb_transport_header(skb) -
3619 			       skb_network_header(skb);
3620 	return hdr_len + skb_gso_transport_seglen(skb);
3621 }
3622 
3623 #endif	/* __KERNEL__ */
3624 #endif	/* _LINUX_SKBUFF_H */
3625