This source file includes following definitions.
- tcp_under_memory_pressure
- before
- between
- tcp_out_of_memory
- tcp_too_many_orphans
- tcp_dec_quickack_mode
- tcp_synq_overflow
- tcp_synq_no_recent_overflow
- tcp_cookie_time
- tcp_clear_xmit_timers
- tcp_bound_to_half_wnd
- tcp_bound_rto
- __tcp_set_rto
- __tcp_fast_path_on
- tcp_fast_path_on
- tcp_fast_path_check
- tcp_rto_min
- tcp_rto_min_us
- tcp_ca_dst_locked
- tcp_min_rtt
- tcp_receive_window
- tcp_clock_ns
- tcp_clock_us
- tcp_time_stamp
- tcp_time_stamp_raw
- tcp_stamp_us_delta
- tcp_skb_timestamp
- tcp_skb_timestamp_us
- bpf_compute_data_end_sk_skb
- tcp_skb_bpf_ingress
- tcp_skb_bpf_redirect_fetch
- tcp_skb_bpf_redirect_clear
- tcp_v6_iif
- tcp_v6_iif_l3_slave
- tcp_v6_sdif
- inet_exact_dif_match
- tcp_v4_sdif
- tcp_skb_pcount
- tcp_skb_pcount_set
- tcp_skb_pcount_add
- tcp_skb_mss
- tcp_skb_can_collapse_to
- tcp_ca_get_name_by_key
- tcp_ca_needs_ecn
- tcp_set_ca_state
- tcp_ca_event
- tcp_is_sack
- tcp_is_reno
- tcp_left_out
- tcp_packets_in_flight
- tcp_in_slow_start
- tcp_in_initial_slowstart
- tcp_in_cwnd_reduction
- tcp_current_ssthresh
- tcp_max_tso_deferred_mss
- tcp_wnd_end
- tcp_is_cwnd_limited
- tcp_needs_internal_pacing
- tcp_pacing_delay
- tcp_reset_xmit_timer
- tcp_probe0_base
- tcp_probe0_when
- tcp_check_probe_timer
- tcp_init_wl
- tcp_update_wl
- tcp_v4_check
- tcp_checksum_complete
- tcp_sack_reset
- tcp_slow_start_after_idle_check
- tcp_win_from_space
- tcp_space
- tcp_full_space
- tcp_rmem_pressure
- keepalive_intvl_when
- keepalive_time_when
- keepalive_probes
- keepalive_time_elapsed
- tcp_fin_time
- tcp_paws_check
- tcp_paws_reject
- tcp_mib_init
- tcp_clear_retrans_hints_partial
- tcp_clear_all_retrans_hints
- tcp_md5_do_lookup
- tcp_md5_do_lookup
- tcp_put_md5sig_pool
- tcp_fastopen_get_ctx
- tcp_fastopen_cookie_match
- tcp_fastopen_context_len
- tcp_skb_tsorted_anchor_cleanup
- tcp_rtx_queue_head
- tcp_rtx_queue_tail
- tcp_write_queue_head
- tcp_write_queue_tail
- tcp_send_head
- tcp_skb_is_last
- tcp_write_queue_empty
- tcp_rtx_queue_empty
- tcp_rtx_and_write_queues_empty
- tcp_add_write_queue_tail
- tcp_insert_write_queue_before
- tcp_unlink_write_queue
- tcp_rtx_queue_unlink
- tcp_rtx_queue_unlink_and_free
- tcp_push_pending_frames
- tcp_highest_sack_seq
- tcp_advance_highest_sack
- tcp_highest_sack
- tcp_highest_sack_reset
- tcp_highest_sack_replace
- inet_sk_transparent
- tcp_stream_is_thin
- tcp_notsent_lowat
- tcp_stream_memory_free
- cookie_init_sequence
- cookie_init_sequence
- tcp_rto_delta_us
- tcp_v4_save_options
- skb_is_tcp_pure_ack
- skb_set_tcp_pure_ack
- tcp_inq
- tcp_segs_in
- tcp_listendrop
- tcp_call_bpf
- tcp_call_bpf_2arg
- tcp_call_bpf_3arg
- tcp_call_bpf
- tcp_call_bpf_2arg
- tcp_call_bpf_3arg
- tcp_timeout_init
- tcp_rwnd_init_bpf
- tcp_bpf_ca_needs_ecn
- tcp_bpf_rtt
- tcp_add_tx_delay
- tcp_transmit_time
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #ifndef _TCP_H
15 #define _TCP_H
16
17 #define FASTRETRANS_DEBUG 1
18
19 #include <linux/list.h>
20 #include <linux/tcp.h>
21 #include <linux/bug.h>
22 #include <linux/slab.h>
23 #include <linux/cache.h>
24 #include <linux/percpu.h>
25 #include <linux/skbuff.h>
26 #include <linux/cryptohash.h>
27 #include <linux/kref.h>
28 #include <linux/ktime.h>
29
30 #include <net/inet_connection_sock.h>
31 #include <net/inet_timewait_sock.h>
32 #include <net/inet_hashtables.h>
33 #include <net/checksum.h>
34 #include <net/request_sock.h>
35 #include <net/sock_reuseport.h>
36 #include <net/sock.h>
37 #include <net/snmp.h>
38 #include <net/ip.h>
39 #include <net/tcp_states.h>
40 #include <net/inet_ecn.h>
41 #include <net/dst.h>
42
43 #include <linux/seq_file.h>
44 #include <linux/memcontrol.h>
45 #include <linux/bpf-cgroup.h>
46 #include <linux/siphash.h>
47
48 extern struct inet_hashinfo tcp_hashinfo;
49
50 extern struct percpu_counter tcp_orphan_count;
51 void tcp_time_wait(struct sock *sk, int state, int timeo);
52
53 #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER)
54 #define MAX_TCP_OPTION_SPACE 40
55 #define TCP_MIN_SND_MSS 48
56 #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
57
58
59
60
61
62 #define MAX_TCP_WINDOW 32767U
63
64
65 #define TCP_MIN_MSS 88U
66
67
68 #define TCP_BASE_MSS 1024
69
70
71 #define TCP_PROBE_INTERVAL 600
72
73
74 #define TCP_PROBE_THRESHOLD 8
75
76
77 #define TCP_FASTRETRANS_THRESH 3
78
79
80 #define TCP_MAX_QUICKACKS 16U
81
82
83 #define TCP_MAX_WSCALE 14U
84
85
86 #define TCP_URG_VALID 0x0100
87 #define TCP_URG_NOTYET 0x0200
88 #define TCP_URG_READ 0x0400
89
90 #define TCP_RETR1 3
91
92
93
94
95
96
97 #define TCP_RETR2 15
98
99
100
101
102
103
104 #define TCP_SYN_RETRIES 6
105
106
107
108
109
110
111
112
113 #define TCP_SYNACK_RETRIES 5
114
115
116
117
118
119
120 #define TCP_TIMEWAIT_LEN (60*HZ)
121
122 #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN
123
124
125
126
127
128
129 #define TCP_DELACK_MAX ((unsigned)(HZ/5))
130 #if HZ >= 100
131 #define TCP_DELACK_MIN ((unsigned)(HZ/25))
132 #define TCP_ATO_MIN ((unsigned)(HZ/25))
133 #else
134 #define TCP_DELACK_MIN 4U
135 #define TCP_ATO_MIN 4U
136 #endif
137 #define TCP_RTO_MAX ((unsigned)(120*HZ))
138 #define TCP_RTO_MIN ((unsigned)(HZ/5))
139 #define TCP_TIMEOUT_MIN (2U)
140 #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ))
141 #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ))
142
143
144
145
146
147
148 #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U))
149
150
151 #define TCP_KEEPALIVE_TIME (120*60*HZ)
152 #define TCP_KEEPALIVE_PROBES 9
153 #define TCP_KEEPALIVE_INTVL (75*HZ)
154
155 #define MAX_TCP_KEEPIDLE 32767
156 #define MAX_TCP_KEEPINTVL 32767
157 #define MAX_TCP_KEEPCNT 127
158 #define MAX_TCP_SYNCNT 127
159
160 #define TCP_SYNQ_INTERVAL (HZ/5)
161
162 #define TCP_PAWS_24DAYS (60 * 60 * 24 * 24)
163 #define TCP_PAWS_MSL 60
164
165
166
167
168
169 #define TCP_PAWS_WINDOW 1
170
171
172
173
174
175
176
177 #define TCPOPT_NOP 1
178 #define TCPOPT_EOL 0
179 #define TCPOPT_MSS 2
180 #define TCPOPT_WINDOW 3
181 #define TCPOPT_SACK_PERM 4
182 #define TCPOPT_SACK 5
183 #define TCPOPT_TIMESTAMP 8
184 #define TCPOPT_MD5SIG 19
185 #define TCPOPT_FASTOPEN 34
186 #define TCPOPT_EXP 254
187
188
189
190 #define TCPOPT_FASTOPEN_MAGIC 0xF989
191 #define TCPOPT_SMC_MAGIC 0xE2D4C3D9
192
193
194
195
196
197 #define TCPOLEN_MSS 4
198 #define TCPOLEN_WINDOW 3
199 #define TCPOLEN_SACK_PERM 2
200 #define TCPOLEN_TIMESTAMP 10
201 #define TCPOLEN_MD5SIG 18
202 #define TCPOLEN_FASTOPEN_BASE 2
203 #define TCPOLEN_EXP_FASTOPEN_BASE 4
204 #define TCPOLEN_EXP_SMC_BASE 6
205
206
207 #define TCPOLEN_TSTAMP_ALIGNED 12
208 #define TCPOLEN_WSCALE_ALIGNED 4
209 #define TCPOLEN_SACKPERM_ALIGNED 4
210 #define TCPOLEN_SACK_BASE 2
211 #define TCPOLEN_SACK_BASE_ALIGNED 4
212 #define TCPOLEN_SACK_PERBLOCK 8
213 #define TCPOLEN_MD5SIG_ALIGNED 20
214 #define TCPOLEN_MSS_ALIGNED 4
215 #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8
216
217
218 #define TCP_NAGLE_OFF 1
219 #define TCP_NAGLE_CORK 2
220 #define TCP_NAGLE_PUSH 4
221
222
223 #define TCP_THIN_LINEAR_RETRIES 6
224
225
226 #define TCP_INIT_CWND 10
227
228
229 #define TFO_CLIENT_ENABLE 1
230 #define TFO_SERVER_ENABLE 2
231 #define TFO_CLIENT_NO_COOKIE 4
232
233
234 #define TFO_SERVER_COOKIE_NOT_REQD 0x200
235
236
237
238
239 #define TFO_SERVER_WO_SOCKOPT1 0x400
240
241
242
243 extern int sysctl_tcp_max_orphans;
244 extern long sysctl_tcp_mem[3];
245
246 #define TCP_RACK_LOSS_DETECTION 0x1
247 #define TCP_RACK_STATIC_REO_WND 0x2
248 #define TCP_RACK_NO_DUPTHRESH 0x4
249
250 extern atomic_long_t tcp_memory_allocated;
251 extern struct percpu_counter tcp_sockets_allocated;
252 extern unsigned long tcp_memory_pressure;
253
254
255 static inline bool tcp_under_memory_pressure(const struct sock *sk)
256 {
257 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
258 mem_cgroup_under_socket_pressure(sk->sk_memcg))
259 return true;
260
261 return READ_ONCE(tcp_memory_pressure);
262 }
263
264
265
266
267
268 static inline bool before(__u32 seq1, __u32 seq2)
269 {
270 return (__s32)(seq1-seq2) < 0;
271 }
272 #define after(seq2, seq1) before(seq1, seq2)
273
274
275 static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
276 {
277 return seq3 - seq2 >= seq1 - seq2;
278 }
279
280 static inline bool tcp_out_of_memory(struct sock *sk)
281 {
282 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
283 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
284 return true;
285 return false;
286 }
287
288 void sk_forced_mem_schedule(struct sock *sk, int size);
289
290 static inline bool tcp_too_many_orphans(struct sock *sk, int shift)
291 {
292 struct percpu_counter *ocp = sk->sk_prot->orphan_count;
293 int orphans = percpu_counter_read_positive(ocp);
294
295 if (orphans << shift > sysctl_tcp_max_orphans) {
296 orphans = percpu_counter_sum_positive(ocp);
297 if (orphans << shift > sysctl_tcp_max_orphans)
298 return true;
299 }
300 return false;
301 }
302
303 bool tcp_check_oom(struct sock *sk, int shift);
304
305
306 extern struct proto tcp_prot;
307
308 #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field)
309 #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field)
310 #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
311 #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
312
313 void tcp_tasklet_init(void);
314
315 int tcp_v4_err(struct sk_buff *skb, u32);
316
317 void tcp_shutdown(struct sock *sk, int how);
318
319 int tcp_v4_early_demux(struct sk_buff *skb);
320 int tcp_v4_rcv(struct sk_buff *skb);
321
322 int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
323 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
324 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
325 int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
326 int flags);
327 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
328 size_t size, int flags);
329 ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
330 size_t size, int flags);
331 void tcp_release_cb(struct sock *sk);
332 void tcp_wfree(struct sk_buff *skb);
333 void tcp_write_timer_handler(struct sock *sk);
334 void tcp_delack_timer_handler(struct sock *sk);
335 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg);
336 int tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
337 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
338 void tcp_rcv_space_adjust(struct sock *sk);
339 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
340 void tcp_twsk_destructor(struct sock *sk);
341 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
342 struct pipe_inode_info *pipe, size_t len,
343 unsigned int flags);
344
345 void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks);
346 static inline void tcp_dec_quickack_mode(struct sock *sk,
347 const unsigned int pkts)
348 {
349 struct inet_connection_sock *icsk = inet_csk(sk);
350
351 if (icsk->icsk_ack.quick) {
352 if (pkts >= icsk->icsk_ack.quick) {
353 icsk->icsk_ack.quick = 0;
354
355 icsk->icsk_ack.ato = TCP_ATO_MIN;
356 } else
357 icsk->icsk_ack.quick -= pkts;
358 }
359 }
360
361 #define TCP_ECN_OK 1
362 #define TCP_ECN_QUEUE_CWR 2
363 #define TCP_ECN_DEMAND_CWR 4
364 #define TCP_ECN_SEEN 8
365
366 enum tcp_tw_status {
367 TCP_TW_SUCCESS = 0,
368 TCP_TW_RST = 1,
369 TCP_TW_ACK = 2,
370 TCP_TW_SYN = 3
371 };
372
373
374 enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw,
375 struct sk_buff *skb,
376 const struct tcphdr *th);
377 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
378 struct request_sock *req, bool fastopen,
379 bool *lost_race);
380 int tcp_child_process(struct sock *parent, struct sock *child,
381 struct sk_buff *skb);
382 void tcp_enter_loss(struct sock *sk);
383 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int flag);
384 void tcp_clear_retrans(struct tcp_sock *tp);
385 void tcp_update_metrics(struct sock *sk);
386 void tcp_init_metrics(struct sock *sk);
387 void tcp_metrics_init(void);
388 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst);
389 void tcp_close(struct sock *sk, long timeout);
390 void tcp_init_sock(struct sock *sk);
391 void tcp_init_transfer(struct sock *sk, int bpf_op);
392 __poll_t tcp_poll(struct file *file, struct socket *sock,
393 struct poll_table_struct *wait);
394 int tcp_getsockopt(struct sock *sk, int level, int optname,
395 char __user *optval, int __user *optlen);
396 int tcp_setsockopt(struct sock *sk, int level, int optname,
397 char __user *optval, unsigned int optlen);
398 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
399 char __user *optval, int __user *optlen);
400 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
401 char __user *optval, unsigned int optlen);
402 void tcp_set_keepalive(struct sock *sk, int val);
403 void tcp_syn_ack_timeout(const struct request_sock *req);
404 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
405 int flags, int *addr_len);
406 int tcp_set_rcvlowat(struct sock *sk, int val);
407 void tcp_data_ready(struct sock *sk);
408 #ifdef CONFIG_MMU
409 int tcp_mmap(struct file *file, struct socket *sock,
410 struct vm_area_struct *vma);
411 #endif
412 void tcp_parse_options(const struct net *net, const struct sk_buff *skb,
413 struct tcp_options_received *opt_rx,
414 int estab, struct tcp_fastopen_cookie *foc);
415 const u8 *tcp_parse_md5sig_option(const struct tcphdr *th);
416
417
418
419
420 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
421 struct tcphdr *th, u32 *cookie);
422 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
423 struct tcphdr *th, u32 *cookie);
424 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
425 const struct tcp_request_sock_ops *af_ops,
426 struct sock *sk, struct tcphdr *th);
427
428
429
430
431 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
432 void tcp_v4_mtu_reduced(struct sock *sk);
433 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
434 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
435 struct sock *tcp_create_openreq_child(const struct sock *sk,
436 struct request_sock *req,
437 struct sk_buff *skb);
438 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
439 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
440 struct request_sock *req,
441 struct dst_entry *dst,
442 struct request_sock *req_unhash,
443 bool *own_req);
444 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
445 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
446 int tcp_connect(struct sock *sk);
447 enum tcp_synack_type {
448 TCP_SYNACK_NORMAL,
449 TCP_SYNACK_FASTOPEN,
450 TCP_SYNACK_COOKIE,
451 };
452 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
453 struct request_sock *req,
454 struct tcp_fastopen_cookie *foc,
455 enum tcp_synack_type synack_type);
456 int tcp_disconnect(struct sock *sk, int flags);
457
458 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
459 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
460 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
461
462
463 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
464 struct request_sock *req,
465 struct dst_entry *dst, u32 tsoff);
466 int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th,
467 u32 cookie);
468 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
469 #ifdef CONFIG_SYN_COOKIES
470
471
472
473
474
475
476
477
478 #define MAX_SYNCOOKIE_AGE 2
479 #define TCP_SYNCOOKIE_PERIOD (60 * HZ)
480 #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD)
481
482
483
484
485
486 static inline void tcp_synq_overflow(const struct sock *sk)
487 {
488 unsigned int last_overflow;
489 unsigned int now = jiffies;
490
491 if (sk->sk_reuseport) {
492 struct sock_reuseport *reuse;
493
494 reuse = rcu_dereference(sk->sk_reuseport_cb);
495 if (likely(reuse)) {
496 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
497 if (!time_between32(now, last_overflow,
498 last_overflow + HZ))
499 WRITE_ONCE(reuse->synq_overflow_ts, now);
500 return;
501 }
502 }
503
504 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
505 if (!time_between32(now, last_overflow, last_overflow + HZ))
506 WRITE_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp, now);
507 }
508
509
510 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk)
511 {
512 unsigned int last_overflow;
513 unsigned int now = jiffies;
514
515 if (sk->sk_reuseport) {
516 struct sock_reuseport *reuse;
517
518 reuse = rcu_dereference(sk->sk_reuseport_cb);
519 if (likely(reuse)) {
520 last_overflow = READ_ONCE(reuse->synq_overflow_ts);
521 return !time_between32(now, last_overflow - HZ,
522 last_overflow +
523 TCP_SYNCOOKIE_VALID);
524 }
525 }
526
527 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);
528
529
530
531
532
533
534
535
536 return !time_between32(now, last_overflow - HZ,
537 last_overflow + TCP_SYNCOOKIE_VALID);
538 }
539
540 static inline u32 tcp_cookie_time(void)
541 {
542 u64 val = get_jiffies_64();
543
544 do_div(val, TCP_SYNCOOKIE_PERIOD);
545 return val;
546 }
547
548 u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th,
549 u16 *mssp);
550 __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss);
551 u64 cookie_init_timestamp(struct request_sock *req);
552 bool cookie_timestamp_decode(const struct net *net,
553 struct tcp_options_received *opt);
554 bool cookie_ecn_ok(const struct tcp_options_received *opt,
555 const struct net *net, const struct dst_entry *dst);
556
557
558 int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th,
559 u32 cookie);
560 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
561
562 u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph,
563 const struct tcphdr *th, u16 *mssp);
564 __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss);
565 #endif
566
567
568 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
569 int nonagle);
570 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
571 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
572 void tcp_retransmit_timer(struct sock *sk);
573 void tcp_xmit_retransmit_queue(struct sock *);
574 void tcp_simple_retransmit(struct sock *);
575 void tcp_enter_recovery(struct sock *sk, bool ece_ack);
576 int tcp_trim_head(struct sock *, struct sk_buff *, u32);
577 enum tcp_queue {
578 TCP_FRAG_IN_WRITE_QUEUE,
579 TCP_FRAG_IN_RTX_QUEUE,
580 };
581 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
582 struct sk_buff *skb, u32 len,
583 unsigned int mss_now, gfp_t gfp);
584
585 void tcp_send_probe0(struct sock *);
586 void tcp_send_partial(struct sock *);
587 int tcp_write_wakeup(struct sock *, int mib);
588 void tcp_send_fin(struct sock *sk);
589 void tcp_send_active_reset(struct sock *sk, gfp_t priority);
590 int tcp_send_synack(struct sock *);
591 void tcp_push_one(struct sock *, unsigned int mss_now);
592 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt);
593 void tcp_send_ack(struct sock *sk);
594 void tcp_send_delayed_ack(struct sock *sk);
595 void tcp_send_loss_probe(struct sock *sk);
596 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
597 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
598 const struct sk_buff *next_skb);
599
600
601 void tcp_rearm_rto(struct sock *sk);
602 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
603 void tcp_reset(struct sock *sk);
604 void tcp_skb_mark_lost_uncond_verify(struct tcp_sock *tp, struct sk_buff *skb);
605 void tcp_fin(struct sock *sk);
606
607
608 void tcp_init_xmit_timers(struct sock *);
609 static inline void tcp_clear_xmit_timers(struct sock *sk)
610 {
611 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)
612 __sock_put(sk);
613
614 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)
615 __sock_put(sk);
616
617 inet_csk_clear_xmit_timers(sk);
618 }
619
620 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
621 unsigned int tcp_current_mss(struct sock *sk);
622
623
624 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
625 {
626 int cutoff;
627
628
629
630
631
632
633
634
635 if (tp->max_window > TCP_MSS_DEFAULT)
636 cutoff = (tp->max_window >> 1);
637 else
638 cutoff = tp->max_window;
639
640 if (cutoff && pktsize > cutoff)
641 return max_t(int, cutoff, 68U - tp->tcp_header_len);
642 else
643 return pktsize;
644 }
645
646
647 void tcp_get_info(struct sock *, struct tcp_info *);
648
649
650 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
651 sk_read_actor_t recv_actor);
652
653 void tcp_initialize_rcv_mss(struct sock *sk);
654
655 int tcp_mtu_to_mss(struct sock *sk, int pmtu);
656 int tcp_mss_to_mtu(struct sock *sk, int mss);
657 void tcp_mtup_init(struct sock *sk);
658 void tcp_init_buffer_space(struct sock *sk);
659
660 static inline void tcp_bound_rto(const struct sock *sk)
661 {
662 if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)
663 inet_csk(sk)->icsk_rto = TCP_RTO_MAX;
664 }
665
666 static inline u32 __tcp_set_rto(const struct tcp_sock *tp)
667 {
668 return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);
669 }
670
671 static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd)
672 {
673 tp->pred_flags = htonl((tp->tcp_header_len << 26) |
674 ntohl(TCP_FLAG_ACK) |
675 snd_wnd);
676 }
677
678 static inline void tcp_fast_path_on(struct tcp_sock *tp)
679 {
680 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);
681 }
682
683 static inline void tcp_fast_path_check(struct sock *sk)
684 {
685 struct tcp_sock *tp = tcp_sk(sk);
686
687 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&
688 tp->rcv_wnd &&
689 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&
690 !tp->urg_data)
691 tcp_fast_path_on(tp);
692 }
693
694
695 static inline u32 tcp_rto_min(struct sock *sk)
696 {
697 const struct dst_entry *dst = __sk_dst_get(sk);
698 u32 rto_min = TCP_RTO_MIN;
699
700 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN))
701 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
702 return rto_min;
703 }
704
705 static inline u32 tcp_rto_min_us(struct sock *sk)
706 {
707 return jiffies_to_usecs(tcp_rto_min(sk));
708 }
709
710 static inline bool tcp_ca_dst_locked(const struct dst_entry *dst)
711 {
712 return dst_metric_locked(dst, RTAX_CC_ALGO);
713 }
714
715
716 static inline u32 tcp_min_rtt(const struct tcp_sock *tp)
717 {
718 return minmax_get(&tp->rtt_min);
719 }
720
721
722
723
724
725 static inline u32 tcp_receive_window(const struct tcp_sock *tp)
726 {
727 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;
728
729 if (win < 0)
730 win = 0;
731 return (u32) win;
732 }
733
734
735
736
737
738 u32 __tcp_select_window(struct sock *sk);
739
740 void tcp_send_window_probe(struct sock *sk);
741
742
743
744
745
746 #define tcp_jiffies32 ((u32)jiffies)
747
748
749
750
751
752
753 #define TCP_TS_HZ 1000
754
755 static inline u64 tcp_clock_ns(void)
756 {
757 return ktime_get_ns();
758 }
759
760 static inline u64 tcp_clock_us(void)
761 {
762 return div_u64(tcp_clock_ns(), NSEC_PER_USEC);
763 }
764
765
766 static inline u32 tcp_time_stamp(const struct tcp_sock *tp)
767 {
768 return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);
769 }
770
771
772 static inline u32 tcp_time_stamp_raw(void)
773 {
774 return div_u64(tcp_clock_ns(), NSEC_PER_SEC / TCP_TS_HZ);
775 }
776
777 void tcp_mstamp_refresh(struct tcp_sock *tp);
778
779 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
780 {
781 return max_t(s64, t1 - t0, 0);
782 }
783
784 static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
785 {
786 return div_u64(skb->skb_mstamp_ns, NSEC_PER_SEC / TCP_TS_HZ);
787 }
788
789
790 static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb)
791 {
792 return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);
793 }
794
795
796 #define tcp_flag_byte(th) (((u_int8_t *)th)[13])
797
798 #define TCPHDR_FIN 0x01
799 #define TCPHDR_SYN 0x02
800 #define TCPHDR_RST 0x04
801 #define TCPHDR_PSH 0x08
802 #define TCPHDR_ACK 0x10
803 #define TCPHDR_URG 0x20
804 #define TCPHDR_ECE 0x40
805 #define TCPHDR_CWR 0x80
806
807 #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR)
808
809
810
811
812
813
814
815 struct tcp_skb_cb {
816 __u32 seq;
817 __u32 end_seq;
818 union {
819
820
821
822
823
824
825 __u32 tcp_tw_isn;
826 struct {
827 u16 tcp_gso_segs;
828 u16 tcp_gso_size;
829 };
830 };
831 __u8 tcp_flags;
832
833 __u8 sacked;
834 #define TCPCB_SACKED_ACKED 0x01
835 #define TCPCB_SACKED_RETRANS 0x02
836 #define TCPCB_LOST 0x04
837 #define TCPCB_TAGBITS 0x07
838 #define TCPCB_REPAIRED 0x10
839 #define TCPCB_EVER_RETRANS 0x80
840 #define TCPCB_RETRANS (TCPCB_SACKED_RETRANS|TCPCB_EVER_RETRANS| \
841 TCPCB_REPAIRED)
842
843 __u8 ip_dsfield;
844 __u8 txstamp_ack:1,
845 eor:1,
846 has_rxtstamp:1,
847 unused:5;
848 __u32 ack_seq;
849 union {
850 struct {
851
852 __u32 in_flight:30,
853 is_app_limited:1,
854 unused:1;
855
856 __u32 delivered;
857
858 u64 first_tx_mstamp;
859
860 u64 delivered_mstamp;
861 } tx;
862 union {
863 struct inet_skb_parm h4;
864 #if IS_ENABLED(CONFIG_IPV6)
865 struct inet6_skb_parm h6;
866 #endif
867 } header;
868 struct {
869 __u32 flags;
870 struct sock *sk_redir;
871 void *data_end;
872 } bpf;
873 };
874 };
875
876 #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0]))
877
878 static inline void bpf_compute_data_end_sk_skb(struct sk_buff *skb)
879 {
880 TCP_SKB_CB(skb)->bpf.data_end = skb->data + skb_headlen(skb);
881 }
882
883 static inline bool tcp_skb_bpf_ingress(const struct sk_buff *skb)
884 {
885 return TCP_SKB_CB(skb)->bpf.flags & BPF_F_INGRESS;
886 }
887
888 static inline struct sock *tcp_skb_bpf_redirect_fetch(struct sk_buff *skb)
889 {
890 return TCP_SKB_CB(skb)->bpf.sk_redir;
891 }
892
893 static inline void tcp_skb_bpf_redirect_clear(struct sk_buff *skb)
894 {
895 TCP_SKB_CB(skb)->bpf.sk_redir = NULL;
896 }
897
898 #if IS_ENABLED(CONFIG_IPV6)
899
900
901
902 static inline int tcp_v6_iif(const struct sk_buff *skb)
903 {
904 return TCP_SKB_CB(skb)->header.h6.iif;
905 }
906
907 static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb)
908 {
909 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
910
911 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;
912 }
913
914
915 static inline int tcp_v6_sdif(const struct sk_buff *skb)
916 {
917 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
918 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))
919 return TCP_SKB_CB(skb)->header.h6.iif;
920 #endif
921 return 0;
922 }
923 #endif
924
925 static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb)
926 {
927 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
928 if (!net->ipv4.sysctl_tcp_l3mdev_accept &&
929 skb && ipv4_l3mdev_skb(IPCB(skb)->flags))
930 return true;
931 #endif
932 return false;
933 }
934
935
936 static inline int tcp_v4_sdif(struct sk_buff *skb)
937 {
938 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
939 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))
940 return TCP_SKB_CB(skb)->header.h4.iif;
941 #endif
942 return 0;
943 }
944
945
946
947
948 static inline int tcp_skb_pcount(const struct sk_buff *skb)
949 {
950 return TCP_SKB_CB(skb)->tcp_gso_segs;
951 }
952
953 static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs)
954 {
955 TCP_SKB_CB(skb)->tcp_gso_segs = segs;
956 }
957
958 static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs)
959 {
960 TCP_SKB_CB(skb)->tcp_gso_segs += segs;
961 }
962
963
964 static inline int tcp_skb_mss(const struct sk_buff *skb)
965 {
966 return TCP_SKB_CB(skb)->tcp_gso_size;
967 }
968
969 static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
970 {
971 return likely(!TCP_SKB_CB(skb)->eor);
972 }
973
974
975 enum tcp_ca_event {
976 CA_EVENT_TX_START,
977 CA_EVENT_CWND_RESTART,
978 CA_EVENT_COMPLETE_CWR,
979 CA_EVENT_LOSS,
980 CA_EVENT_ECN_NO_CE,
981 CA_EVENT_ECN_IS_CE,
982 };
983
984
985 enum tcp_ca_ack_event_flags {
986 CA_ACK_SLOWPATH = (1 << 0),
987 CA_ACK_WIN_UPDATE = (1 << 1),
988 CA_ACK_ECE = (1 << 2),
989 };
990
991
992
993
994 #define TCP_CA_NAME_MAX 16
995 #define TCP_CA_MAX 128
996 #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX)
997
998 #define TCP_CA_UNSPEC 0
999
1000
1001 #define TCP_CONG_NON_RESTRICTED 0x1
1002
1003 #define TCP_CONG_NEEDS_ECN 0x2
1004
1005 union tcp_cc_info;
1006
1007 struct ack_sample {
1008 u32 pkts_acked;
1009 s32 rtt_us;
1010 u32 in_flight;
1011 };
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021 struct rate_sample {
1022 u64 prior_mstamp;
1023 u32 prior_delivered;
1024 s32 delivered;
1025 long interval_us;
1026 u32 snd_interval_us;
1027 u32 rcv_interval_us;
1028 long rtt_us;
1029 int losses;
1030 u32 acked_sacked;
1031 u32 prior_in_flight;
1032 bool is_app_limited;
1033 bool is_retrans;
1034 bool is_ack_delayed;
1035 };
1036
1037 struct tcp_congestion_ops {
1038 struct list_head list;
1039 u32 key;
1040 u32 flags;
1041
1042
1043 void (*init)(struct sock *sk);
1044
1045 void (*release)(struct sock *sk);
1046
1047
1048 u32 (*ssthresh)(struct sock *sk);
1049
1050 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1051
1052 void (*set_state)(struct sock *sk, u8 new_state);
1053
1054 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1055
1056 void (*in_ack_event)(struct sock *sk, u32 flags);
1057
1058 u32 (*undo_cwnd)(struct sock *sk);
1059
1060 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1061
1062 u32 (*min_tso_segs)(struct sock *sk);
1063
1064 u32 (*sndbuf_expand)(struct sock *sk);
1065
1066
1067
1068 void (*cong_control)(struct sock *sk, const struct rate_sample *rs);
1069
1070 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1071 union tcp_cc_info *info);
1072
1073 char name[TCP_CA_NAME_MAX];
1074 struct module *owner;
1075 };
1076
1077 int tcp_register_congestion_control(struct tcp_congestion_ops *type);
1078 void tcp_unregister_congestion_control(struct tcp_congestion_ops *type);
1079
1080 void tcp_assign_congestion_control(struct sock *sk);
1081 void tcp_init_congestion_control(struct sock *sk);
1082 void tcp_cleanup_congestion_control(struct sock *sk);
1083 int tcp_set_default_congestion_control(struct net *net, const char *name);
1084 void tcp_get_default_congestion_control(struct net *net, char *name);
1085 void tcp_get_available_congestion_control(char *buf, size_t len);
1086 void tcp_get_allowed_congestion_control(char *buf, size_t len);
1087 int tcp_set_allowed_congestion_control(char *allowed);
1088 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1089 bool reinit, bool cap_net_admin);
1090 u32 tcp_slow_start(struct tcp_sock *tp, u32 acked);
1091 void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked);
1092
1093 u32 tcp_reno_ssthresh(struct sock *sk);
1094 u32 tcp_reno_undo_cwnd(struct sock *sk);
1095 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1096 extern struct tcp_congestion_ops tcp_reno;
1097
1098 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1099 u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
1100 #ifdef CONFIG_INET
1101 char *tcp_ca_get_name_by_key(u32 key, char *buffer);
1102 #else
1103 static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
1104 {
1105 return NULL;
1106 }
1107 #endif
1108
1109 static inline bool tcp_ca_needs_ecn(const struct sock *sk)
1110 {
1111 const struct inet_connection_sock *icsk = inet_csk(sk);
1112
1113 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;
1114 }
1115
1116 static inline void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
1117 {
1118 struct inet_connection_sock *icsk = inet_csk(sk);
1119
1120 if (icsk->icsk_ca_ops->set_state)
1121 icsk->icsk_ca_ops->set_state(sk, ca_state);
1122 icsk->icsk_ca_state = ca_state;
1123 }
1124
1125 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
1126 {
1127 const struct inet_connection_sock *icsk = inet_csk(sk);
1128
1129 if (icsk->icsk_ca_ops->cwnd_event)
1130 icsk->icsk_ca_ops->cwnd_event(sk, event);
1131 }
1132
1133
1134 void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1135 void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1136 struct rate_sample *rs);
1137 void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1138 bool is_sack_reneg, struct rate_sample *rs);
1139 void tcp_rate_check_app_limited(struct sock *sk);
1140
1141
1142
1143
1144
1145
1146
1147
1148 static inline int tcp_is_sack(const struct tcp_sock *tp)
1149 {
1150 return likely(tp->rx_opt.sack_ok);
1151 }
1152
1153 static inline bool tcp_is_reno(const struct tcp_sock *tp)
1154 {
1155 return !tcp_is_sack(tp);
1156 }
1157
1158 static inline unsigned int tcp_left_out(const struct tcp_sock *tp)
1159 {
1160 return tp->sacked_out + tp->lost_out;
1161 }
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177 static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
1178 {
1179 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
1180 }
1181
1182 #define TCP_INFINITE_SSTHRESH 0x7fffffff
1183
1184 static inline bool tcp_in_slow_start(const struct tcp_sock *tp)
1185 {
1186 return tp->snd_cwnd < tp->snd_ssthresh;
1187 }
1188
1189 static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp)
1190 {
1191 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;
1192 }
1193
1194 static inline bool tcp_in_cwnd_reduction(const struct sock *sk)
1195 {
1196 return (TCPF_CA_CWR | TCPF_CA_Recovery) &
1197 (1 << inet_csk(sk)->icsk_ca_state);
1198 }
1199
1200
1201
1202
1203
1204 static inline __u32 tcp_current_ssthresh(const struct sock *sk)
1205 {
1206 const struct tcp_sock *tp = tcp_sk(sk);
1207
1208 if (tcp_in_cwnd_reduction(sk))
1209 return tp->snd_ssthresh;
1210 else
1211 return max(tp->snd_ssthresh,
1212 ((tp->snd_cwnd >> 1) +
1213 (tp->snd_cwnd >> 2)));
1214 }
1215
1216
1217 #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out)
1218
1219 void tcp_enter_cwr(struct sock *sk);
1220 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst);
1221
1222
1223
1224
1225 static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp)
1226 {
1227 return 3;
1228 }
1229
1230
1231 static inline u32 tcp_wnd_end(const struct tcp_sock *tp)
1232 {
1233 return tp->snd_una + tp->snd_wnd;
1234 }
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249 static inline bool tcp_is_cwnd_limited(const struct sock *sk)
1250 {
1251 const struct tcp_sock *tp = tcp_sk(sk);
1252
1253
1254 if (tcp_in_slow_start(tp))
1255 return tp->snd_cwnd < 2 * tp->max_packets_out;
1256
1257 return tp->is_cwnd_limited;
1258 }
1259
1260
1261
1262
1263
1264
1265
1266 static inline bool tcp_needs_internal_pacing(const struct sock *sk)
1267 {
1268 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
1269 }
1270
1271
1272
1273
1274 static inline unsigned long tcp_pacing_delay(const struct sock *sk,
1275 const struct sk_buff *skb)
1276 {
1277 s64 pacing_delay = skb ? skb->tstamp : tcp_sk(sk)->tcp_wstamp_ns;
1278
1279 pacing_delay -= tcp_sk(sk)->tcp_clock_cache;
1280
1281 return pacing_delay > 0 ? nsecs_to_jiffies(pacing_delay) : 0;
1282 }
1283
1284 static inline void tcp_reset_xmit_timer(struct sock *sk,
1285 const int what,
1286 unsigned long when,
1287 const unsigned long max_when,
1288 const struct sk_buff *skb)
1289 {
1290 inet_csk_reset_xmit_timer(sk, what, when + tcp_pacing_delay(sk, skb),
1291 max_when);
1292 }
1293
1294
1295
1296
1297
1298
1299
1300 static inline unsigned long tcp_probe0_base(const struct sock *sk)
1301 {
1302 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);
1303 }
1304
1305
1306 static inline unsigned long tcp_probe0_when(const struct sock *sk,
1307 unsigned long max_when)
1308 {
1309 u64 when = (u64)tcp_probe0_base(sk) << inet_csk(sk)->icsk_backoff;
1310
1311 return (unsigned long)min_t(u64, when, max_when);
1312 }
1313
1314 static inline void tcp_check_probe_timer(struct sock *sk)
1315 {
1316 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)
1317 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
1318 tcp_probe0_base(sk), TCP_RTO_MAX,
1319 NULL);
1320 }
1321
1322 static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq)
1323 {
1324 tp->snd_wl1 = seq;
1325 }
1326
1327 static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq)
1328 {
1329 tp->snd_wl1 = seq;
1330 }
1331
1332
1333
1334
1335 static inline __sum16 tcp_v4_check(int len, __be32 saddr,
1336 __be32 daddr, __wsum base)
1337 {
1338 return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base);
1339 }
1340
1341 static inline bool tcp_checksum_complete(struct sk_buff *skb)
1342 {
1343 return !skb_csum_unnecessary(skb) &&
1344 __skb_checksum_complete(skb);
1345 }
1346
1347 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb);
1348 int tcp_filter(struct sock *sk, struct sk_buff *skb);
1349 void tcp_set_state(struct sock *sk, int state);
1350 void tcp_done(struct sock *sk);
1351 int tcp_abort(struct sock *sk, int err);
1352
1353 static inline void tcp_sack_reset(struct tcp_options_received *rx_opt)
1354 {
1355 rx_opt->dsack = 0;
1356 rx_opt->num_sacks = 0;
1357 }
1358
1359 u32 tcp_default_init_rwnd(u32 mss);
1360 void tcp_cwnd_restart(struct sock *sk, s32 delta);
1361
1362 static inline void tcp_slow_start_after_idle_check(struct sock *sk)
1363 {
1364 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1365 struct tcp_sock *tp = tcp_sk(sk);
1366 s32 delta;
1367
1368 if (!sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle || tp->packets_out ||
1369 ca_ops->cong_control)
1370 return;
1371 delta = tcp_jiffies32 - tp->lsndtime;
1372 if (delta > inet_csk(sk)->icsk_rto)
1373 tcp_cwnd_restart(sk, delta);
1374 }
1375
1376
1377 void tcp_select_initial_window(const struct sock *sk, int __space,
1378 __u32 mss, __u32 *rcv_wnd,
1379 __u32 *window_clamp, int wscale_ok,
1380 __u8 *rcv_wscale, __u32 init_rcv_wnd);
1381
1382 static inline int tcp_win_from_space(const struct sock *sk, int space)
1383 {
1384 int tcp_adv_win_scale = sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale;
1385
1386 return tcp_adv_win_scale <= 0 ?
1387 (space>>(-tcp_adv_win_scale)) :
1388 space - (space>>tcp_adv_win_scale);
1389 }
1390
1391
1392 static inline int tcp_space(const struct sock *sk)
1393 {
1394 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
1395 READ_ONCE(sk->sk_backlog.len) -
1396 atomic_read(&sk->sk_rmem_alloc));
1397 }
1398
1399 static inline int tcp_full_space(const struct sock *sk)
1400 {
1401 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1402 }
1403
1404
1405
1406
1407
1408
1409 static inline bool tcp_rmem_pressure(const struct sock *sk)
1410 {
1411 int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1412 int threshold = rcvbuf - (rcvbuf >> 3);
1413
1414 return atomic_read(&sk->sk_rmem_alloc) > threshold;
1415 }
1416
1417 extern void tcp_openreq_init_rwin(struct request_sock *req,
1418 const struct sock *sk_listener,
1419 const struct dst_entry *dst);
1420
1421 void tcp_enter_memory_pressure(struct sock *sk);
1422 void tcp_leave_memory_pressure(struct sock *sk);
1423
1424 static inline int keepalive_intvl_when(const struct tcp_sock *tp)
1425 {
1426 struct net *net = sock_net((struct sock *)tp);
1427
1428 return tp->keepalive_intvl ? : net->ipv4.sysctl_tcp_keepalive_intvl;
1429 }
1430
1431 static inline int keepalive_time_when(const struct tcp_sock *tp)
1432 {
1433 struct net *net = sock_net((struct sock *)tp);
1434
1435 return tp->keepalive_time ? : net->ipv4.sysctl_tcp_keepalive_time;
1436 }
1437
1438 static inline int keepalive_probes(const struct tcp_sock *tp)
1439 {
1440 struct net *net = sock_net((struct sock *)tp);
1441
1442 return tp->keepalive_probes ? : net->ipv4.sysctl_tcp_keepalive_probes;
1443 }
1444
1445 static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp)
1446 {
1447 const struct inet_connection_sock *icsk = &tp->inet_conn;
1448
1449 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,
1450 tcp_jiffies32 - tp->rcv_tstamp);
1451 }
1452
1453 static inline int tcp_fin_time(const struct sock *sk)
1454 {
1455 int fin_timeout = tcp_sk(sk)->linger2 ? : sock_net(sk)->ipv4.sysctl_tcp_fin_timeout;
1456 const int rto = inet_csk(sk)->icsk_rto;
1457
1458 if (fin_timeout < (rto << 2) - (rto >> 1))
1459 fin_timeout = (rto << 2) - (rto >> 1);
1460
1461 return fin_timeout;
1462 }
1463
1464 static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt,
1465 int paws_win)
1466 {
1467 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)
1468 return true;
1469 if (unlikely(!time_before32(ktime_get_seconds(),
1470 rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))
1471 return true;
1472
1473
1474
1475
1476
1477 if (!rx_opt->ts_recent)
1478 return true;
1479 return false;
1480 }
1481
1482 static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt,
1483 int rst)
1484 {
1485 if (tcp_paws_check(rx_opt, 0))
1486 return false;
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500 if (rst && !time_before32(ktime_get_seconds(),
1501 rx_opt->ts_recent_stamp + TCP_PAWS_MSL))
1502 return false;
1503 return true;
1504 }
1505
1506 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb,
1507 int mib_idx, u32 *last_oow_ack_time);
1508
1509 static inline void tcp_mib_init(struct net *net)
1510 {
1511
1512 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1);
1513 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ);
1514 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ);
1515 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);
1516 }
1517
1518
1519 static inline void tcp_clear_retrans_hints_partial(struct tcp_sock *tp)
1520 {
1521 tp->lost_skb_hint = NULL;
1522 }
1523
1524 static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp)
1525 {
1526 tcp_clear_retrans_hints_partial(tp);
1527 tp->retransmit_skb_hint = NULL;
1528 }
1529
1530 union tcp_md5_addr {
1531 struct in_addr a4;
1532 #if IS_ENABLED(CONFIG_IPV6)
1533 struct in6_addr a6;
1534 #endif
1535 };
1536
1537
1538 struct tcp_md5sig_key {
1539 struct hlist_node node;
1540 u8 keylen;
1541 u8 family;
1542 union tcp_md5_addr addr;
1543 u8 prefixlen;
1544 u8 key[TCP_MD5SIG_MAXKEYLEN];
1545 struct rcu_head rcu;
1546 };
1547
1548
1549 struct tcp_md5sig_info {
1550 struct hlist_head head;
1551 struct rcu_head rcu;
1552 };
1553
1554
1555 struct tcp4_pseudohdr {
1556 __be32 saddr;
1557 __be32 daddr;
1558 __u8 pad;
1559 __u8 protocol;
1560 __be16 len;
1561 };
1562
1563 struct tcp6_pseudohdr {
1564 struct in6_addr saddr;
1565 struct in6_addr daddr;
1566 __be32 len;
1567 __be32 protocol;
1568 };
1569
1570 union tcp_md5sum_block {
1571 struct tcp4_pseudohdr ip4;
1572 #if IS_ENABLED(CONFIG_IPV6)
1573 struct tcp6_pseudohdr ip6;
1574 #endif
1575 };
1576
1577
1578 struct tcp_md5sig_pool {
1579 struct ahash_request *md5_req;
1580 void *scratch;
1581 };
1582
1583
1584 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1585 const struct sock *sk, const struct sk_buff *skb);
1586 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1587 int family, u8 prefixlen, const u8 *newkey, u8 newkeylen,
1588 gfp_t gfp);
1589 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1590 int family, u8 prefixlen);
1591 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1592 const struct sock *addr_sk);
1593
1594 #ifdef CONFIG_TCP_MD5SIG
1595 #include <linux/jump_label.h>
1596 extern struct static_key_false tcp_md5_needed;
1597 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,
1598 const union tcp_md5_addr *addr,
1599 int family);
1600 static inline struct tcp_md5sig_key *
1601 tcp_md5_do_lookup(const struct sock *sk,
1602 const union tcp_md5_addr *addr,
1603 int family)
1604 {
1605 if (!static_branch_unlikely(&tcp_md5_needed))
1606 return NULL;
1607 return __tcp_md5_do_lookup(sk, addr, family);
1608 }
1609
1610 #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
1611 #else
1612 static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
1613 const union tcp_md5_addr *addr,
1614 int family)
1615 {
1616 return NULL;
1617 }
1618 #define tcp_twsk_md5_key(twsk) NULL
1619 #endif
1620
1621 bool tcp_alloc_md5sig_pool(void);
1622
1623 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void);
1624 static inline void tcp_put_md5sig_pool(void)
1625 {
1626 local_bh_enable();
1627 }
1628
1629 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *, const struct sk_buff *,
1630 unsigned int header_len);
1631 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp,
1632 const struct tcp_md5sig_key *key);
1633
1634
1635 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1636 struct tcp_fastopen_cookie *cookie);
1637 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1638 struct tcp_fastopen_cookie *cookie, bool syn_lost,
1639 u16 try_exp);
1640 struct tcp_fastopen_request {
1641
1642 struct tcp_fastopen_cookie cookie;
1643 struct msghdr *data;
1644 size_t size;
1645 int copied;
1646 struct ubuf_info *uarg;
1647 };
1648 void tcp_free_fastopen_req(struct tcp_sock *tp);
1649 void tcp_fastopen_destroy_cipher(struct sock *sk);
1650 void tcp_fastopen_ctx_destroy(struct net *net);
1651 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1652 void *primary_key, void *backup_key);
1653 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1654 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1655 struct request_sock *req,
1656 struct tcp_fastopen_cookie *foc,
1657 const struct dst_entry *dst);
1658 void tcp_fastopen_init_key_once(struct net *net);
1659 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1660 struct tcp_fastopen_cookie *cookie);
1661 bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
1662 #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t)
1663 #define TCP_FASTOPEN_KEY_MAX 2
1664 #define TCP_FASTOPEN_KEY_BUF_LENGTH \
1665 (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX)
1666
1667
1668 struct tcp_fastopen_context {
1669 siphash_key_t key[TCP_FASTOPEN_KEY_MAX];
1670 int num;
1671 struct rcu_head rcu;
1672 };
1673
1674 extern unsigned int sysctl_tcp_fastopen_blackhole_timeout;
1675 void tcp_fastopen_active_disable(struct sock *sk);
1676 bool tcp_fastopen_active_should_disable(struct sock *sk);
1677 void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
1678 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
1679
1680
1681 static inline
1682 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk)
1683 {
1684 struct tcp_fastopen_context *ctx;
1685
1686 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);
1687 if (!ctx)
1688 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);
1689 return ctx;
1690 }
1691
1692 static inline
1693 bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc,
1694 const struct tcp_fastopen_cookie *orig)
1695 {
1696 if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&
1697 orig->len == foc->len &&
1698 !memcmp(orig->val, foc->val, foc->len))
1699 return true;
1700 return false;
1701 }
1702
1703 static inline
1704 int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx)
1705 {
1706 return ctx->num;
1707 }
1708
1709
1710
1711
1712 enum tcp_chrono {
1713 TCP_CHRONO_UNSPEC,
1714 TCP_CHRONO_BUSY,
1715 TCP_CHRONO_RWND_LIMITED,
1716 TCP_CHRONO_SNDBUF_LIMITED,
1717 __TCP_CHRONO_MAX,
1718 };
1719
1720 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
1721 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
1722
1723
1724
1725
1726 static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb)
1727 {
1728 skb->destructor = NULL;
1729 skb->_skb_refdst = 0UL;
1730 }
1731
1732 #define tcp_skb_tsorted_save(skb) { \
1733 unsigned long _save = skb->_skb_refdst; \
1734 skb->_skb_refdst = 0UL;
1735
1736 #define tcp_skb_tsorted_restore(skb) \
1737 skb->_skb_refdst = _save; \
1738 }
1739
1740 void tcp_write_queue_purge(struct sock *sk);
1741
1742 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk)
1743 {
1744 return skb_rb_first(&sk->tcp_rtx_queue);
1745 }
1746
1747 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk)
1748 {
1749 return skb_rb_last(&sk->tcp_rtx_queue);
1750 }
1751
1752 static inline struct sk_buff *tcp_write_queue_head(const struct sock *sk)
1753 {
1754 return skb_peek(&sk->sk_write_queue);
1755 }
1756
1757 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk)
1758 {
1759 return skb_peek_tail(&sk->sk_write_queue);
1760 }
1761
1762 #define tcp_for_write_queue_from_safe(skb, tmp, sk) \
1763 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1764
1765 static inline struct sk_buff *tcp_send_head(const struct sock *sk)
1766 {
1767 return skb_peek(&sk->sk_write_queue);
1768 }
1769
1770 static inline bool tcp_skb_is_last(const struct sock *sk,
1771 const struct sk_buff *skb)
1772 {
1773 return skb_queue_is_last(&sk->sk_write_queue, skb);
1774 }
1775
1776 static inline bool tcp_write_queue_empty(const struct sock *sk)
1777 {
1778 return skb_queue_empty(&sk->sk_write_queue);
1779 }
1780
1781 static inline bool tcp_rtx_queue_empty(const struct sock *sk)
1782 {
1783 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);
1784 }
1785
1786 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk)
1787 {
1788 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk);
1789 }
1790
1791 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb)
1792 {
1793 __skb_queue_tail(&sk->sk_write_queue, skb);
1794
1795
1796 if (sk->sk_write_queue.next == skb)
1797 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
1798 }
1799
1800
1801 static inline void tcp_insert_write_queue_before(struct sk_buff *new,
1802 struct sk_buff *skb,
1803 struct sock *sk)
1804 {
1805 __skb_queue_before(&sk->sk_write_queue, skb, new);
1806 }
1807
1808 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk)
1809 {
1810 tcp_skb_tsorted_anchor_cleanup(skb);
1811 __skb_unlink(skb, &sk->sk_write_queue);
1812 }
1813
1814 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb);
1815
1816 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk)
1817 {
1818 tcp_skb_tsorted_anchor_cleanup(skb);
1819 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);
1820 }
1821
1822 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk)
1823 {
1824 list_del(&skb->tcp_tsorted_anchor);
1825 tcp_rtx_queue_unlink(skb, sk);
1826 sk_wmem_free_skb(sk, skb);
1827 }
1828
1829 static inline void tcp_push_pending_frames(struct sock *sk)
1830 {
1831 if (tcp_send_head(sk)) {
1832 struct tcp_sock *tp = tcp_sk(sk);
1833
1834 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);
1835 }
1836 }
1837
1838
1839
1840
1841
1842 static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp)
1843 {
1844 if (!tp->sacked_out)
1845 return tp->snd_una;
1846
1847 if (tp->highest_sack == NULL)
1848 return tp->snd_nxt;
1849
1850 return TCP_SKB_CB(tp->highest_sack)->seq;
1851 }
1852
1853 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb)
1854 {
1855 tcp_sk(sk)->highest_sack = skb_rb_next(skb);
1856 }
1857
1858 static inline struct sk_buff *tcp_highest_sack(struct sock *sk)
1859 {
1860 return tcp_sk(sk)->highest_sack;
1861 }
1862
1863 static inline void tcp_highest_sack_reset(struct sock *sk)
1864 {
1865 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);
1866 }
1867
1868
1869 static inline void tcp_highest_sack_replace(struct sock *sk,
1870 struct sk_buff *old,
1871 struct sk_buff *new)
1872 {
1873 if (old == tcp_highest_sack(sk))
1874 tcp_sk(sk)->highest_sack = new;
1875 }
1876
1877
1878 static inline bool inet_sk_transparent(const struct sock *sk)
1879 {
1880 switch (sk->sk_state) {
1881 case TCP_TIME_WAIT:
1882 return inet_twsk(sk)->tw_transparent;
1883 case TCP_NEW_SYN_RECV:
1884 return inet_rsk(inet_reqsk(sk))->no_srccheck;
1885 }
1886 return inet_sk(sk)->transparent;
1887 }
1888
1889
1890
1891
1892 static inline bool tcp_stream_is_thin(struct tcp_sock *tp)
1893 {
1894 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);
1895 }
1896
1897
1898 enum tcp_seq_states {
1899 TCP_SEQ_STATE_LISTENING,
1900 TCP_SEQ_STATE_ESTABLISHED,
1901 };
1902
1903 void *tcp_seq_start(struct seq_file *seq, loff_t *pos);
1904 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos);
1905 void tcp_seq_stop(struct seq_file *seq, void *v);
1906
1907 struct tcp_seq_afinfo {
1908 sa_family_t family;
1909 };
1910
1911 struct tcp_iter_state {
1912 struct seq_net_private p;
1913 enum tcp_seq_states state;
1914 struct sock *syn_wait_sk;
1915 int bucket, offset, sbucket, num;
1916 loff_t last_pos;
1917 };
1918
1919 extern struct request_sock_ops tcp_request_sock_ops;
1920 extern struct request_sock_ops tcp6_request_sock_ops;
1921
1922 void tcp_v4_destroy_sock(struct sock *sk);
1923
1924 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
1925 netdev_features_t features);
1926 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
1927 int tcp_gro_complete(struct sk_buff *skb);
1928
1929 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);
1930
1931 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
1932 {
1933 struct net *net = sock_net((struct sock *)tp);
1934 return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
1935 }
1936
1937
1938
1939
1940
1941 static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)
1942 {
1943 const struct tcp_sock *tp = tcp_sk(sk);
1944 u32 notsent_bytes = READ_ONCE(tp->write_seq) -
1945 READ_ONCE(tp->snd_nxt);
1946
1947 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
1948 }
1949
1950 #ifdef CONFIG_PROC_FS
1951 int tcp4_proc_init(void);
1952 void tcp4_proc_exit(void);
1953 #endif
1954
1955 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
1956 int tcp_conn_request(struct request_sock_ops *rsk_ops,
1957 const struct tcp_request_sock_ops *af_ops,
1958 struct sock *sk, struct sk_buff *skb);
1959
1960
1961 struct tcp_sock_af_ops {
1962 #ifdef CONFIG_TCP_MD5SIG
1963 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
1964 const struct sock *addr_sk);
1965 int (*calc_md5_hash)(char *location,
1966 const struct tcp_md5sig_key *md5,
1967 const struct sock *sk,
1968 const struct sk_buff *skb);
1969 int (*md5_parse)(struct sock *sk,
1970 int optname,
1971 char __user *optval,
1972 int optlen);
1973 #endif
1974 };
1975
1976 struct tcp_request_sock_ops {
1977 u16 mss_clamp;
1978 #ifdef CONFIG_TCP_MD5SIG
1979 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
1980 const struct sock *addr_sk);
1981 int (*calc_md5_hash) (char *location,
1982 const struct tcp_md5sig_key *md5,
1983 const struct sock *sk,
1984 const struct sk_buff *skb);
1985 #endif
1986 void (*init_req)(struct request_sock *req,
1987 const struct sock *sk_listener,
1988 struct sk_buff *skb);
1989 #ifdef CONFIG_SYN_COOKIES
1990 __u32 (*cookie_init_seq)(const struct sk_buff *skb,
1991 __u16 *mss);
1992 #endif
1993 struct dst_entry *(*route_req)(const struct sock *sk, struct flowi *fl,
1994 const struct request_sock *req);
1995 u32 (*init_seq)(const struct sk_buff *skb);
1996 u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb);
1997 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
1998 struct flowi *fl, struct request_sock *req,
1999 struct tcp_fastopen_cookie *foc,
2000 enum tcp_synack_type synack_type);
2001 };
2002
2003 #ifdef CONFIG_SYN_COOKIES
2004 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2005 const struct sock *sk, struct sk_buff *skb,
2006 __u16 *mss)
2007 {
2008 tcp_synq_overflow(sk);
2009 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT);
2010 return ops->cookie_init_seq(skb, mss);
2011 }
2012 #else
2013 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops,
2014 const struct sock *sk, struct sk_buff *skb,
2015 __u16 *mss)
2016 {
2017 return 0;
2018 }
2019 #endif
2020
2021 int tcpv4_offload_init(void);
2022
2023 void tcp_v4_init(void);
2024 void tcp_init(void);
2025
2026
2027 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2028 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2029 extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb,
2030 u32 reo_wnd);
2031 extern void tcp_rack_mark_lost(struct sock *sk);
2032 extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq,
2033 u64 xmit_time);
2034 extern void tcp_rack_reo_timeout(struct sock *sk);
2035 extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2036
2037
2038 static inline s64 tcp_rto_delta_us(const struct sock *sk)
2039 {
2040 const struct sk_buff *skb = tcp_rtx_queue_head(sk);
2041 u32 rto = inet_csk(sk)->icsk_rto;
2042 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto);
2043
2044 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;
2045 }
2046
2047
2048
2049
2050 static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net,
2051 struct sk_buff *skb)
2052 {
2053 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;
2054 struct ip_options_rcu *dopt = NULL;
2055
2056 if (opt->optlen) {
2057 int opt_size = sizeof(*dopt) + opt->optlen;
2058
2059 dopt = kmalloc(opt_size, GFP_ATOMIC);
2060 if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {
2061 kfree(dopt);
2062 dopt = NULL;
2063 }
2064 }
2065 return dopt;
2066 }
2067
2068
2069
2070
2071
2072
2073 static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb)
2074 {
2075 return skb->truesize == 2;
2076 }
2077
2078 static inline void skb_set_tcp_pure_ack(struct sk_buff *skb)
2079 {
2080 skb->truesize = 2;
2081 }
2082
2083 static inline int tcp_inq(struct sock *sk)
2084 {
2085 struct tcp_sock *tp = tcp_sk(sk);
2086 int answ;
2087
2088 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
2089 answ = 0;
2090 } else if (sock_flag(sk, SOCK_URGINLINE) ||
2091 !tp->urg_data ||
2092 before(tp->urg_seq, tp->copied_seq) ||
2093 !before(tp->urg_seq, tp->rcv_nxt)) {
2094
2095 answ = tp->rcv_nxt - tp->copied_seq;
2096
2097
2098 if (answ && sock_flag(sk, SOCK_DONE))
2099 answ--;
2100 } else {
2101 answ = tp->urg_seq - tp->copied_seq;
2102 }
2103
2104 return answ;
2105 }
2106
2107 int tcp_peek_len(struct socket *sock);
2108
2109 static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
2110 {
2111 u16 segs_in;
2112
2113 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2114 tp->segs_in += segs_in;
2115 if (skb->len > tcp_hdrlen(skb))
2116 tp->data_segs_in += segs_in;
2117 }
2118
2119
2120
2121
2122
2123
2124
2125
2126 static inline void tcp_listendrop(const struct sock *sk)
2127 {
2128 atomic_inc(&((struct sock *)sk)->sk_drops);
2129 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS);
2130 }
2131
2132 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer);
2133
2134
2135
2136
2137
2138 #define TCP_ULP_NAME_MAX 16
2139 #define TCP_ULP_MAX 128
2140 #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX)
2141
2142 struct tcp_ulp_ops {
2143 struct list_head list;
2144
2145
2146 int (*init)(struct sock *sk);
2147
2148 void (*update)(struct sock *sk, struct proto *p,
2149 void (*write_space)(struct sock *sk));
2150
2151 void (*release)(struct sock *sk);
2152
2153 int (*get_info)(const struct sock *sk, struct sk_buff *skb);
2154 size_t (*get_info_size)(const struct sock *sk);
2155
2156 char name[TCP_ULP_NAME_MAX];
2157 struct module *owner;
2158 };
2159 int tcp_register_ulp(struct tcp_ulp_ops *type);
2160 void tcp_unregister_ulp(struct tcp_ulp_ops *type);
2161 int tcp_set_ulp(struct sock *sk, const char *name);
2162 void tcp_get_available_ulp(char *buf, size_t len);
2163 void tcp_cleanup_ulp(struct sock *sk);
2164 void tcp_update_ulp(struct sock *sk, struct proto *p,
2165 void (*write_space)(struct sock *sk));
2166
2167 #define MODULE_ALIAS_TCP_ULP(name) \
2168 __MODULE_INFO(alias, alias_userspace, name); \
2169 __MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2170
2171 struct sk_msg;
2172 struct sk_psock;
2173
2174 int tcp_bpf_init(struct sock *sk);
2175 void tcp_bpf_reinit(struct sock *sk);
2176 int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
2177 int flags);
2178 int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
2179 int nonblock, int flags, int *addr_len);
2180 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
2181 struct msghdr *msg, int len, int flags);
2182
2183
2184
2185
2186
2187
2188 #ifdef CONFIG_BPF
2189 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2190 {
2191 struct bpf_sock_ops_kern sock_ops;
2192 int ret;
2193
2194 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
2195 if (sk_fullsock(sk)) {
2196 sock_ops.is_fullsock = 1;
2197 sock_owned_by_me(sk);
2198 }
2199
2200 sock_ops.sk = sk;
2201 sock_ops.op = op;
2202 if (nargs > 0)
2203 memcpy(sock_ops.args, args, nargs * sizeof(*args));
2204
2205 ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
2206 if (ret == 0)
2207 ret = sock_ops.reply;
2208 else
2209 ret = -1;
2210 return ret;
2211 }
2212
2213 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2214 {
2215 u32 args[2] = {arg1, arg2};
2216
2217 return tcp_call_bpf(sk, op, 2, args);
2218 }
2219
2220 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2221 u32 arg3)
2222 {
2223 u32 args[3] = {arg1, arg2, arg3};
2224
2225 return tcp_call_bpf(sk, op, 3, args);
2226 }
2227
2228 #else
2229 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
2230 {
2231 return -EPERM;
2232 }
2233
2234 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
2235 {
2236 return -EPERM;
2237 }
2238
2239 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
2240 u32 arg3)
2241 {
2242 return -EPERM;
2243 }
2244
2245 #endif
2246
2247 static inline u32 tcp_timeout_init(struct sock *sk)
2248 {
2249 int timeout;
2250
2251 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
2252
2253 if (timeout <= 0)
2254 timeout = TCP_TIMEOUT_INIT;
2255 return timeout;
2256 }
2257
2258 static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
2259 {
2260 int rwnd;
2261
2262 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
2263
2264 if (rwnd < 0)
2265 rwnd = 0;
2266 return rwnd;
2267 }
2268
2269 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
2270 {
2271 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
2272 }
2273
2274 static inline void tcp_bpf_rtt(struct sock *sk)
2275 {
2276 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
2277 tcp_call_bpf(sk, BPF_SOCK_OPS_RTT_CB, 0, NULL);
2278 }
2279
2280 #if IS_ENABLED(CONFIG_SMC)
2281 extern struct static_key_false tcp_have_smc;
2282 #endif
2283
2284 #if IS_ENABLED(CONFIG_TLS_DEVICE)
2285 void clean_acked_data_enable(struct inet_connection_sock *icsk,
2286 void (*cad)(struct sock *sk, u32 ack_seq));
2287 void clean_acked_data_disable(struct inet_connection_sock *icsk);
2288 void clean_acked_data_flush(void);
2289 #endif
2290
2291 DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2292 static inline void tcp_add_tx_delay(struct sk_buff *skb,
2293 const struct tcp_sock *tp)
2294 {
2295 if (static_branch_unlikely(&tcp_tx_delay_enabled))
2296 skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;
2297 }
2298
2299
2300
2301
2302 static inline u64 tcp_transmit_time(const struct sock *sk)
2303 {
2304 if (static_branch_unlikely(&tcp_tx_delay_enabled)) {
2305 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?
2306 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;
2307
2308 return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;
2309 }
2310 return 0;
2311 }
2312
2313 #endif