383 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_init_sock()  local
385 	__skb_queue_head_init(&tp->out_of_order_queue);  in tcp_init_sock()
387 	tcp_prequeue_init(tp);  in tcp_init_sock()
388 	INIT_LIST_HEAD(&tp->tsq_node);  in tcp_init_sock()
391 	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);  in tcp_init_sock()
398 	tp->snd_cwnd = TCP_INIT_CWND;  in tcp_init_sock()
403 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;  in tcp_init_sock()
404 	tp->snd_cwnd_clamp = ~0;  in tcp_init_sock()
405 	tp->mss_cache = TCP_MSS_DEFAULT;  in tcp_init_sock()
406 	u64_stats_init(&tp->syncp);  in tcp_init_sock()
408 	tp->reordering = sysctl_tcp_reordering;  in tcp_init_sock()
409 	tcp_enable_early_retrans(tp);  in tcp_init_sock()
412 	tp->tsoffset = 0;  in tcp_init_sock()
453 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_poll()  local
502 	    (sk->sk_state != TCP_SYN_RECV || tp->fastopen_rsk)) {  in tcp_poll()
505 		if (tp->urg_seq == tp->copied_seq &&  in tcp_poll()
507 		    tp->urg_data)  in tcp_poll()
513 		if (tp->rcv_nxt - tp->copied_seq >= target)  in tcp_poll()
536 		if (tp->urg_data & TCP_URG_VALID)  in tcp_poll()
550 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_ioctl()  local
563 			 !tp->urg_data ||  in tcp_ioctl()
564 			 before(tp->urg_seq, tp->copied_seq) ||  in tcp_ioctl()
565 			 !before(tp->urg_seq, tp->rcv_nxt)) {  in tcp_ioctl()
567 			answ = tp->rcv_nxt - tp->copied_seq;  in tcp_ioctl()
573 			answ = tp->urg_seq - tp->copied_seq;  in tcp_ioctl()
577 		answ = tp->urg_data && tp->urg_seq == tp->copied_seq;  in tcp_ioctl()
586 			answ = tp->write_seq - tp->snd_una;  in tcp_ioctl()
595 			answ = tp->write_seq - tp->snd_nxt;  in tcp_ioctl()
605 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)  in tcp_mark_push()  argument
608 	tp->pushed_seq = tp->write_seq;  in tcp_mark_push()
611 static inline bool forced_push(const struct tcp_sock *tp)  in forced_push()  argument
613 	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));  in forced_push()
618 	struct tcp_sock *tp = tcp_sk(sk);  in skb_entail()  local
622 	tcb->seq     = tcb->end_seq = tp->write_seq;  in skb_entail()
629 	if (tp->nonagle & TCP_NAGLE_PUSH)  in skb_entail()
630 		tp->nonagle &= ~TCP_NAGLE_PUSH;  in skb_entail()
633 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)  in tcp_mark_urg()  argument
636 		tp->snd_up = tp->write_seq;  in tcp_mark_urg()
661 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_push()  local
668 	if (!(flags & MSG_MORE) || forced_push(tp))  in tcp_push()
669 		tcp_mark_push(tp, skb);  in tcp_push()
671 	tcp_mark_urg(tp, flags);  in tcp_push()
676 		if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {  in tcp_push()
678 			set_bit(TSQ_THROTTLED, &tp->tsq_flags);  in tcp_push()
842 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_xmit_size_goal()  local
850 	new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);  in tcp_xmit_size_goal()
853 	size_goal = tp->gso_segs * mss_now;  in tcp_xmit_size_goal()
856 		tp->gso_segs = min_t(u16, new_size_goal / mss_now,  in tcp_xmit_size_goal()
858 		size_goal = tp->gso_segs * mss_now;  in tcp_xmit_size_goal()
877 	struct tcp_sock *tp = tcp_sk(sk);  in do_tcp_sendpages()  local
926 			tcp_mark_push(tp, skb);  in do_tcp_sendpages()
946 		tp->write_seq += copy;  in do_tcp_sendpages()
963 		if (forced_push(tp)) {  in do_tcp_sendpages()
964 			tcp_mark_push(tp, skb);  in do_tcp_sendpages()
984 		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);  in do_tcp_sendpages()
1013 	const struct tcp_sock *tp = tcp_sk(sk);  in select_size()  local
1014 	int tmp = tp->mss_cache;  in select_size()
1034 void tcp_free_fastopen_req(struct tcp_sock *tp)  in tcp_free_fastopen_req()  argument
1036 	if (tp->fastopen_req) {  in tcp_free_fastopen_req()
1037 		kfree(tp->fastopen_req);  in tcp_free_fastopen_req()
1038 		tp->fastopen_req = NULL;  in tcp_free_fastopen_req()
1045 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_sendmsg_fastopen()  local
1050 	if (tp->fastopen_req)  in tcp_sendmsg_fastopen()
1053 	tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),  in tcp_sendmsg_fastopen()
1055 	if (unlikely(!tp->fastopen_req))  in tcp_sendmsg_fastopen()
1057 	tp->fastopen_req->data = msg;  in tcp_sendmsg_fastopen()
1058 	tp->fastopen_req->size = size;  in tcp_sendmsg_fastopen()
1063 	*copied = tp->fastopen_req->copied;  in tcp_sendmsg_fastopen()
1064 	tcp_free_fastopen_req(tp);  in tcp_sendmsg_fastopen()
1070 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_sendmsg()  local
1100 	if (unlikely(tp->repair)) {  in tcp_sendmsg()
1101 		if (tp->repair_queue == TCP_RECV_QUEUE) {  in tcp_sendmsg()
1107 		if (tp->repair_queue == TCP_NO_QUEUE)  in tcp_sendmsg()
1166 			if (tp->repair)  in tcp_sendmsg()
1192 					tcp_mark_push(tp, skb);  in tcp_sendmsg()
1224 		tp->write_seq += copy;  in tcp_sendmsg()
1234 		if (skb->len < max || (flags & MSG_OOB) || unlikely(tp->repair))  in tcp_sendmsg()
1237 		if (forced_push(tp)) {  in tcp_sendmsg()
1238 			tcp_mark_push(tp, skb);  in tcp_sendmsg()
1259 		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);  in tcp_sendmsg()
1291 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_recv_urg()  local
1294 	if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||  in tcp_recv_urg()
1295 	    tp->urg_data == TCP_URG_READ)  in tcp_recv_urg()
1301 	if (tp->urg_data & TCP_URG_VALID) {  in tcp_recv_urg()
1303 		char c = tp->urg_data;  in tcp_recv_urg()
1306 			tp->urg_data = TCP_URG_READ;  in tcp_recv_urg()
1359 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_cleanup_rbuf()  local
1364 	WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),  in tcp_cleanup_rbuf()
1366 	     tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);  in tcp_cleanup_rbuf()
1374 		    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||  in tcp_cleanup_rbuf()
1396 		__u32 rcv_window_now = tcp_receive_window(tp);  in tcp_cleanup_rbuf()
1399 		if (2*rcv_window_now <= tp->window_clamp) {  in tcp_cleanup_rbuf()
1418 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_prequeue_process()  local
1425 	while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)  in tcp_prequeue_process()
1430 	tp->ucopy.memory = 0;  in tcp_prequeue_process()
1470 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_read_sock()  local
1471 	u32 seq = tp->copied_seq;  in tcp_read_sock()
1484 			if (tp->urg_data) {  in tcp_read_sock()
1485 				u32 urg_offset = tp->urg_seq - seq;  in tcp_read_sock()
1523 		tp->copied_seq = seq;  in tcp_read_sock()
1525 	tp->copied_seq = seq;  in tcp_read_sock()
1549 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_recvmsg()  local
1580 	if (unlikely(tp->repair)) {  in tcp_recvmsg()
1585 		if (tp->repair_queue == TCP_SEND_QUEUE)  in tcp_recvmsg()
1589 		if (tp->repair_queue == TCP_NO_QUEUE)  in tcp_recvmsg()
1595 	seq = &tp->copied_seq;  in tcp_recvmsg()
1597 		peek_seq = tp->copied_seq;  in tcp_recvmsg()
1607 		if (tp->urg_data && tp->urg_seq == *seq) {  in tcp_recvmsg()
1624 				 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,  in tcp_recvmsg()
1637 			     *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);  in tcp_recvmsg()
1688 		if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {  in tcp_recvmsg()
1692 				tp->ucopy.task = user_recv;  in tcp_recvmsg()
1693 				tp->ucopy.msg = msg;  in tcp_recvmsg()
1696 			tp->ucopy.len = len;  in tcp_recvmsg()
1698 			WARN_ON(tp->copied_seq != tp->rcv_nxt &&  in tcp_recvmsg()
1727 			if (!skb_queue_empty(&tp->ucopy.prequeue))  in tcp_recvmsg()
1745 			if ((chunk = len - tp->ucopy.len) != 0) {  in tcp_recvmsg()
1751 			if (tp->rcv_nxt == tp->copied_seq &&  in tcp_recvmsg()
1752 			    !skb_queue_empty(&tp->ucopy.prequeue)) {  in tcp_recvmsg()
1756 				if ((chunk = len - tp->ucopy.len) != 0) {  in tcp_recvmsg()
1764 		    (peek_seq - copied - urg_hole != tp->copied_seq)) {  in tcp_recvmsg()
1768 			peek_seq = tp->copied_seq;  in tcp_recvmsg()
1779 		if (tp->urg_data) {  in tcp_recvmsg()
1780 			u32 urg_offset = tp->urg_seq - *seq;  in tcp_recvmsg()
1813 		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {  in tcp_recvmsg()
1814 			tp->urg_data = 0;  in tcp_recvmsg()
1835 		if (!skb_queue_empty(&tp->ucopy.prequeue)) {  in tcp_recvmsg()
1838 			tp->ucopy.len = copied > 0 ? len : 0;  in tcp_recvmsg()
1842 			if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {  in tcp_recvmsg()
1849 		tp->ucopy.task = NULL;  in tcp_recvmsg()
1850 		tp->ucopy.len = 0;  in tcp_recvmsg()
2112 		struct tcp_sock *tp = tcp_sk(sk);  in tcp_close()  local
2113 		if (tp->linger2 < 0) {  in tcp_close()
2172 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_disconnect()  local
2182 	} else if (unlikely(tp->repair)) {  in tcp_disconnect()
2185 		   (tp->snd_nxt != tp->write_seq &&  in tcp_disconnect()
2198 	__skb_queue_purge(&tp->out_of_order_queue);  in tcp_disconnect()
2207 	tp->srtt_us = 0;  in tcp_disconnect()
2208 	if ((tp->write_seq += tp->max_window + 2) == 0)  in tcp_disconnect()
2209 		tp->write_seq = 1;  in tcp_disconnect()
2211 	tp->snd_cwnd = 2;  in tcp_disconnect()
2213 	tp->packets_out = 0;  in tcp_disconnect()
2214 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;  in tcp_disconnect()
2215 	tp->snd_cwnd_cnt = 0;  in tcp_disconnect()
2216 	tp->window_clamp = 0;  in tcp_disconnect()
2218 	tcp_clear_retrans(tp);  in tcp_disconnect()
2221 	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));  in tcp_disconnect()
2244 static int tcp_repair_options_est(struct tcp_sock *tp,  in tcp_repair_options_est()  argument
2258 			tp->rx_opt.mss_clamp = opt.opt_val;  in tcp_repair_options_est()
2268 				tp->rx_opt.snd_wscale = snd_wscale;  in tcp_repair_options_est()
2269 				tp->rx_opt.rcv_wscale = rcv_wscale;  in tcp_repair_options_est()
2270 				tp->rx_opt.wscale_ok = 1;  in tcp_repair_options_est()
2277 			tp->rx_opt.sack_ok |= TCP_SACK_SEEN;  in tcp_repair_options_est()
2279 				tcp_enable_fack(tp);  in tcp_repair_options_est()
2285 			tp->rx_opt.tstamp_ok = 1;  in tcp_repair_options_est()
2299 	struct tcp_sock *tp = tcp_sk(sk);  in do_tcp_setsockopt()  local
2345 		tp->rx_opt.user_mss = val;  in do_tcp_setsockopt()
2358 			tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;  in do_tcp_setsockopt()
2361 			tp->nonagle &= ~TCP_NAGLE_OFF;  in do_tcp_setsockopt()
2369 			tp->thin_lto = val;  in do_tcp_setsockopt()
2376 			tp->thin_dupack = val;  in do_tcp_setsockopt()
2377 			if (tp->thin_dupack)  in do_tcp_setsockopt()
2378 				tcp_disable_early_retrans(tp);  in do_tcp_setsockopt()
2386 			tp->repair = 1;  in do_tcp_setsockopt()
2388 			tp->repair_queue = TCP_NO_QUEUE;  in do_tcp_setsockopt()
2390 			tp->repair = 0;  in do_tcp_setsockopt()
2399 		if (!tp->repair)  in do_tcp_setsockopt()
2402 			tp->repair_queue = val;  in do_tcp_setsockopt()
2410 		else if (tp->repair_queue == TCP_SEND_QUEUE)  in do_tcp_setsockopt()
2411 			tp->write_seq = val;  in do_tcp_setsockopt()
2412 		else if (tp->repair_queue == TCP_RECV_QUEUE)  in do_tcp_setsockopt()
2413 			tp->rcv_nxt = val;  in do_tcp_setsockopt()
2419 		if (!tp->repair)  in do_tcp_setsockopt()
2422 			err = tcp_repair_options_est(tp,  in do_tcp_setsockopt()
2442 			tp->nonagle |= TCP_NAGLE_CORK;  in do_tcp_setsockopt()
2444 			tp->nonagle &= ~TCP_NAGLE_CORK;  in do_tcp_setsockopt()
2445 			if (tp->nonagle&TCP_NAGLE_OFF)  in do_tcp_setsockopt()
2446 				tp->nonagle |= TCP_NAGLE_PUSH;  in do_tcp_setsockopt()
2455 			tp->keepalive_time = val * HZ;  in do_tcp_setsockopt()
2459 				u32 elapsed = keepalive_time_elapsed(tp);  in do_tcp_setsockopt()
2460 				if (tp->keepalive_time > elapsed)  in do_tcp_setsockopt()
2461 					elapsed = tp->keepalive_time - elapsed;  in do_tcp_setsockopt()
2472 			tp->keepalive_intvl = val * HZ;  in do_tcp_setsockopt()
2478 			tp->keepalive_probes = val;  in do_tcp_setsockopt()
2489 			tp->linger2 = -1;  in do_tcp_setsockopt()
2491 			tp->linger2 = 0;  in do_tcp_setsockopt()
2493 			tp->linger2 = val * HZ;  in do_tcp_setsockopt()
2509 			tp->window_clamp = 0;  in do_tcp_setsockopt()
2511 			tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?  in do_tcp_setsockopt()
2534 		err = tp->af_specific->md5_parse(sk, optval, optlen);  in do_tcp_setsockopt()
2558 		if (!tp->repair)  in do_tcp_setsockopt()
2561 			tp->tsoffset = val - tcp_time_stamp;  in do_tcp_setsockopt()
2564 		tp->notsent_lowat = val;  in do_tcp_setsockopt()
2603 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_get_info()  local
2618 	if (tp->rx_opt.tstamp_ok)  in tcp_get_info()
2620 	if (tcp_is_sack(tp))  in tcp_get_info()
2622 	if (tp->rx_opt.wscale_ok) {  in tcp_get_info()
2624 		info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;  in tcp_get_info()
2625 		info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;  in tcp_get_info()
2628 	if (tp->ecn_flags & TCP_ECN_OK)  in tcp_get_info()
2630 	if (tp->ecn_flags & TCP_ECN_SEEN)  in tcp_get_info()
2632 	if (tp->syn_data_acked)  in tcp_get_info()
2637 	info->tcpi_snd_mss = tp->mss_cache;  in tcp_get_info()
2644 		info->tcpi_unacked = tp->packets_out;  in tcp_get_info()
2645 		info->tcpi_sacked = tp->sacked_out;  in tcp_get_info()
2647 	info->tcpi_lost = tp->lost_out;  in tcp_get_info()
2648 	info->tcpi_retrans = tp->retrans_out;  in tcp_get_info()
2649 	info->tcpi_fackets = tp->fackets_out;  in tcp_get_info()
2651 	info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);  in tcp_get_info()
2653 	info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);  in tcp_get_info()
2656 	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;  in tcp_get_info()
2657 	info->tcpi_rtt = tp->srtt_us >> 3;  in tcp_get_info()
2658 	info->tcpi_rttvar = tp->mdev_us >> 2;  in tcp_get_info()
2659 	info->tcpi_snd_ssthresh = tp->snd_ssthresh;  in tcp_get_info()
2660 	info->tcpi_snd_cwnd = tp->snd_cwnd;  in tcp_get_info()
2661 	info->tcpi_advmss = tp->advmss;  in tcp_get_info()
2662 	info->tcpi_reordering = tp->reordering;  in tcp_get_info()
2664 	info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;  in tcp_get_info()
2665 	info->tcpi_rcv_space = tp->rcvq_space.space;  in tcp_get_info()
2667 	info->tcpi_total_retrans = tp->total_retrans;  in tcp_get_info()
2678 		start = u64_stats_fetch_begin_irq(&tp->syncp);  in tcp_get_info()
2679 		put_unaligned(tp->bytes_acked, &info->tcpi_bytes_acked);  in tcp_get_info()
2680 		put_unaligned(tp->bytes_received, &info->tcpi_bytes_received);  in tcp_get_info()
2681 	} while (u64_stats_fetch_retry_irq(&tp->syncp, start));  in tcp_get_info()
2689 	struct tcp_sock *tp = tcp_sk(sk);  in do_tcp_getsockopt()  local
2702 		val = tp->mss_cache;  in do_tcp_getsockopt()
2704 			val = tp->rx_opt.user_mss;  in do_tcp_getsockopt()
2705 		if (tp->repair)  in do_tcp_getsockopt()
2706 			val = tp->rx_opt.mss_clamp;  in do_tcp_getsockopt()
2709 		val = !!(tp->nonagle&TCP_NAGLE_OFF);  in do_tcp_getsockopt()
2712 		val = !!(tp->nonagle&TCP_NAGLE_CORK);  in do_tcp_getsockopt()
2715 		val = keepalive_time_when(tp) / HZ;  in do_tcp_getsockopt()
2718 		val = keepalive_intvl_when(tp) / HZ;  in do_tcp_getsockopt()
2721 		val = keepalive_probes(tp);  in do_tcp_getsockopt()
2727 		val = tp->linger2;  in do_tcp_getsockopt()
2736 		val = tp->window_clamp;  in do_tcp_getsockopt()
2788 		val = tp->thin_lto;  in do_tcp_getsockopt()
2791 		val = tp->thin_dupack;  in do_tcp_getsockopt()
2795 		val = tp->repair;  in do_tcp_getsockopt()
2799 		if (tp->repair)  in do_tcp_getsockopt()
2800 			val = tp->repair_queue;  in do_tcp_getsockopt()
2806 		if (tp->repair_queue == TCP_SEND_QUEUE)  in do_tcp_getsockopt()
2807 			val = tp->write_seq;  in do_tcp_getsockopt()
2808 		else if (tp->repair_queue == TCP_RECV_QUEUE)  in do_tcp_getsockopt()
2809 			val = tp->rcv_nxt;  in do_tcp_getsockopt()
2826 		val = tcp_time_stamp + tp->tsoffset;  in do_tcp_getsockopt()
2829 		val = tp->notsent_lowat;  in do_tcp_getsockopt()
2950 	const struct tcphdr *tp = tcp_hdr(skb);  in tcp_md5_hash_skb_data()  local
2960 	sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);  in tcp_md5_hash_skb_data()