57 static inline struct net *tm_net(struct tcp_metrics_block *tm)  in tm_net()  argument
59 	return read_pnet(&tm->tcpm_net);  in tm_net()
62 static bool tcp_metric_locked(struct tcp_metrics_block *tm,  in tcp_metric_locked()  argument
65 	return tm->tcpm_lock & (1 << idx);  in tcp_metric_locked()
68 static u32 tcp_metric_get(struct tcp_metrics_block *tm,  in tcp_metric_get()  argument
71 	return tm->tcpm_vals[idx];  in tcp_metric_get()
74 static void tcp_metric_set(struct tcp_metrics_block *tm,  in tcp_metric_set()  argument
78 	tm->tcpm_vals[idx] = val;  in tcp_metric_set()
100 static void tcpm_suck_dst(struct tcp_metrics_block *tm,  in tcpm_suck_dst()  argument
107 	tm->tcpm_stamp = jiffies;  in tcpm_suck_dst()
120 	tm->tcpm_lock = val;  in tcpm_suck_dst()
123 	tm->tcpm_vals[TCP_METRIC_RTT] = msval * USEC_PER_MSEC;  in tcpm_suck_dst()
126 	tm->tcpm_vals[TCP_METRIC_RTTVAR] = msval * USEC_PER_MSEC;  in tcpm_suck_dst()
127 	tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH);  in tcpm_suck_dst()
128 	tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND);  in tcpm_suck_dst()
129 	tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING);  in tcpm_suck_dst()
130 	tm->tcpm_ts = 0;  in tcpm_suck_dst()
131 	tm->tcpm_ts_stamp = 0;  in tcpm_suck_dst()
133 		tm->tcpm_fastopen.mss = 0;  in tcpm_suck_dst()
134 		tm->tcpm_fastopen.syn_loss = 0;  in tcpm_suck_dst()
135 		tm->tcpm_fastopen.try_exp = 0;  in tcpm_suck_dst()
136 		tm->tcpm_fastopen.cookie.exp = false;  in tcpm_suck_dst()
137 		tm->tcpm_fastopen.cookie.len = 0;  in tcpm_suck_dst()
143 static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst)  in tcpm_check_stamp()  argument
145 	if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT)))  in tcpm_check_stamp()
146 		tcpm_suck_dst(tm, dst, false);  in tcpm_check_stamp()
160 	struct tcp_metrics_block *tm;  in tcpm_new()  local
170 	tm = __tcp_get_metrics(saddr, daddr, net, hash);  in tcpm_new()
171 	if (tm == TCP_METRICS_RECLAIM_PTR) {  in tcpm_new()
173 		tm = NULL;  in tcpm_new()
175 	if (tm) {  in tcpm_new()
176 		tcpm_check_stamp(tm, dst);  in tcpm_new()
184 		for (tm = deref_locked(oldest->tcpm_next); tm;  in tcpm_new()
185 		     tm = deref_locked(tm->tcpm_next)) {  in tcpm_new()
186 			if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp))  in tcpm_new()
187 				oldest = tm;  in tcpm_new()
189 		tm = oldest;  in tcpm_new()
191 		tm = kmalloc(sizeof(*tm), GFP_ATOMIC);  in tcpm_new()
192 		if (!tm)  in tcpm_new()
195 	write_pnet(&tm->tcpm_net, net);  in tcpm_new()
196 	tm->tcpm_saddr = *saddr;  in tcpm_new()
197 	tm->tcpm_daddr = *daddr;  in tcpm_new()
199 	tcpm_suck_dst(tm, dst, true);  in tcpm_new()
202 		tm->tcpm_next = tcp_metrics_hash[hash].chain;  in tcpm_new()
203 		rcu_assign_pointer(tcp_metrics_hash[hash].chain, tm);  in tcpm_new()
208 	return tm;  in tcpm_new()
211 static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth)  in tcp_get_encode()  argument
213 	if (tm)  in tcp_get_encode()
214 		return tm;  in tcp_get_encode()
224 	struct tcp_metrics_block *tm;  in __tcp_get_metrics()  local
227 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in __tcp_get_metrics()
228 	     tm = rcu_dereference(tm->tcpm_next)) {  in __tcp_get_metrics()
229 		if (addr_same(&tm->tcpm_saddr, saddr) &&  in __tcp_get_metrics()
230 		    addr_same(&tm->tcpm_daddr, daddr) &&  in __tcp_get_metrics()
231 		    net_eq(tm_net(tm), net))  in __tcp_get_metrics()
235 	return tcp_get_encode(tm, depth);  in __tcp_get_metrics()
241 	struct tcp_metrics_block *tm;  in __tcp_get_metrics_req()  local
269 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in __tcp_get_metrics_req()
270 	     tm = rcu_dereference(tm->tcpm_next)) {  in __tcp_get_metrics_req()
271 		if (addr_same(&tm->tcpm_saddr, &saddr) &&  in __tcp_get_metrics_req()
272 		    addr_same(&tm->tcpm_daddr, &daddr) &&  in __tcp_get_metrics_req()
273 		    net_eq(tm_net(tm), net))  in __tcp_get_metrics_req()
276 	tcpm_check_stamp(tm, dst);  in __tcp_get_metrics_req()
277 	return tm;  in __tcp_get_metrics_req()
282 	struct tcp_metrics_block *tm;  in __tcp_get_metrics_tw()  local
318 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in __tcp_get_metrics_tw()
319 	     tm = rcu_dereference(tm->tcpm_next)) {  in __tcp_get_metrics_tw()
320 		if (addr_same(&tm->tcpm_saddr, &saddr) &&  in __tcp_get_metrics_tw()
321 		    addr_same(&tm->tcpm_daddr, &daddr) &&  in __tcp_get_metrics_tw()
322 		    net_eq(tm_net(tm), net))  in __tcp_get_metrics_tw()
325 	return tm;  in __tcp_get_metrics_tw()
332 	struct tcp_metrics_block *tm;  in tcp_get_metrics()  local
368 	tm = __tcp_get_metrics(&saddr, &daddr, net, hash);  in tcp_get_metrics()
369 	if (tm == TCP_METRICS_RECLAIM_PTR)  in tcp_get_metrics()
370 		tm = NULL;  in tcp_get_metrics()
371 	if (!tm && create)  in tcp_get_metrics()
372 		tm = tcpm_new(dst, &saddr, &daddr, hash);  in tcp_get_metrics()
374 		tcpm_check_stamp(tm, dst);  in tcp_get_metrics()
376 	return tm;  in tcp_get_metrics()
388 	struct tcp_metrics_block *tm;  in tcp_update_metrics()  local
405 		tm = tcp_get_metrics(sk, dst, false);  in tcp_update_metrics()
406 		if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT))  in tcp_update_metrics()
407 			tcp_metric_set(tm, TCP_METRIC_RTT, 0);  in tcp_update_metrics()
410 		tm = tcp_get_metrics(sk, dst, true);  in tcp_update_metrics()
412 	if (!tm)  in tcp_update_metrics()
415 	rtt = tcp_metric_get(tm, TCP_METRIC_RTT);  in tcp_update_metrics()
422 	if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) {  in tcp_update_metrics()
427 		tcp_metric_set(tm, TCP_METRIC_RTT, rtt);  in tcp_update_metrics()
430 	if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) {  in tcp_update_metrics()
441 		var = tcp_metric_get(tm, TCP_METRIC_RTTVAR);  in tcp_update_metrics()
447 		tcp_metric_set(tm, TCP_METRIC_RTTVAR, var);  in tcp_update_metrics()
452 		if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {  in tcp_update_metrics()
453 			val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);  in tcp_update_metrics()
455 				tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcp_update_metrics()
458 		if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {  in tcp_update_metrics()
459 			val = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_update_metrics()
461 				tcp_metric_set(tm, TCP_METRIC_CWND,  in tcp_update_metrics()
467 		if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH))  in tcp_update_metrics()
468 			tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcp_update_metrics()
470 		if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {  in tcp_update_metrics()
471 			val = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_update_metrics()
472 			tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1);  in tcp_update_metrics()
478 		if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {  in tcp_update_metrics()
479 			val = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_update_metrics()
480 			tcp_metric_set(tm, TCP_METRIC_CWND,  in tcp_update_metrics()
483 		if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {  in tcp_update_metrics()
484 			val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);  in tcp_update_metrics()
486 				tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcp_update_metrics()
489 		if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) {  in tcp_update_metrics()
490 			val = tcp_metric_get(tm, TCP_METRIC_REORDERING);  in tcp_update_metrics()
493 				tcp_metric_set(tm, TCP_METRIC_REORDERING,  in tcp_update_metrics()
497 	tm->tcpm_stamp = jiffies;  in tcp_update_metrics()
508 	struct tcp_metrics_block *tm;  in tcp_init_metrics()  local
517 	tm = tcp_get_metrics(sk, dst, true);  in tcp_init_metrics()
518 	if (!tm) {  in tcp_init_metrics()
523 	if (tcp_metric_locked(tm, TCP_METRIC_CWND))  in tcp_init_metrics()
524 		tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_init_metrics()
526 	val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);  in tcp_init_metrics()
537 	val = tcp_metric_get(tm, TCP_METRIC_REORDERING);  in tcp_init_metrics()
544 	crtt = tcp_metric_get(tm, TCP_METRIC_RTT);  in tcp_init_metrics()
598 	struct tcp_metrics_block *tm;  in tcp_peer_is_proven()  local
605 	tm = __tcp_get_metrics_req(req, dst);  in tcp_peer_is_proven()
607 		if (tm &&  in tcp_peer_is_proven()
608 		    (u32)get_seconds() - tm->tcpm_ts_stamp < TCP_PAWS_MSL &&  in tcp_peer_is_proven()
609 		    ((s32)(tm->tcpm_ts - req->ts_recent) > TCP_PAWS_WINDOW ||  in tcp_peer_is_proven()
615 		if (tm && tcp_metric_get(tm, TCP_METRIC_RTT) && tm->tcpm_ts_stamp)  in tcp_peer_is_proven()
628 	struct tcp_metrics_block *tm;  in tcp_fetch_timewait_stamp()  local
631 	tm = tcp_get_metrics(sk, dst, true);  in tcp_fetch_timewait_stamp()
632 	if (tm) {  in tcp_fetch_timewait_stamp()
635 		if ((u32)get_seconds() - tm->tcpm_ts_stamp <= TCP_PAWS_MSL) {  in tcp_fetch_timewait_stamp()
636 			tp->rx_opt.ts_recent_stamp = tm->tcpm_ts_stamp;  in tcp_fetch_timewait_stamp()
637 			tp->rx_opt.ts_recent = tm->tcpm_ts;  in tcp_fetch_timewait_stamp()
655 		struct tcp_metrics_block *tm;  in tcp_remember_stamp()  local
658 		tm = tcp_get_metrics(sk, dst, true);  in tcp_remember_stamp()
659 		if (tm) {  in tcp_remember_stamp()
662 			if ((s32)(tm->tcpm_ts - tp->rx_opt.ts_recent) <= 0 ||  in tcp_remember_stamp()
663 			    ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&  in tcp_remember_stamp()
664 			     tm->tcpm_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) {  in tcp_remember_stamp()
665 				tm->tcpm_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp;  in tcp_remember_stamp()
666 				tm->tcpm_ts = tp->rx_opt.ts_recent;  in tcp_remember_stamp()
677 	struct tcp_metrics_block *tm;  in tcp_tw_remember_stamp()  local
681 	tm = __tcp_get_metrics_tw(tw);  in tcp_tw_remember_stamp()
682 	if (tm) {  in tcp_tw_remember_stamp()
687 		if ((s32)(tm->tcpm_ts - tcptw->tw_ts_recent) <= 0 ||  in tcp_tw_remember_stamp()
688 		    ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&  in tcp_tw_remember_stamp()
689 		     tm->tcpm_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) {  in tcp_tw_remember_stamp()
690 			tm->tcpm_ts_stamp = (u32)tcptw->tw_ts_recent_stamp;  in tcp_tw_remember_stamp()
691 			tm->tcpm_ts	   = tcptw->tw_ts_recent;  in tcp_tw_remember_stamp()
706 	struct tcp_metrics_block *tm;  in tcp_fastopen_cache_get()  local
709 	tm = tcp_get_metrics(sk, __sk_dst_get(sk), false);  in tcp_fastopen_cache_get()
710 	if (tm) {  in tcp_fastopen_cache_get()
711 		struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;  in tcp_fastopen_cache_get()
733 	struct tcp_metrics_block *tm;  in tcp_fastopen_cache_set()  local
738 	tm = tcp_get_metrics(sk, dst, true);  in tcp_fastopen_cache_set()
739 	if (tm) {  in tcp_fastopen_cache_set()
740 		struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;  in tcp_fastopen_cache_set()
791 				 struct tcp_metrics_block *tm)  in tcp_metrics_fill_info()  argument
796 	switch (tm->tcpm_daddr.family) {  in tcp_metrics_fill_info()
799 				    tm->tcpm_daddr.addr.a4) < 0)  in tcp_metrics_fill_info()
802 				    tm->tcpm_saddr.addr.a4) < 0)  in tcp_metrics_fill_info()
807 				     &tm->tcpm_daddr.addr.in6) < 0)  in tcp_metrics_fill_info()
810 				     &tm->tcpm_saddr.addr.in6) < 0)  in tcp_metrics_fill_info()
818 			  jiffies - tm->tcpm_stamp) < 0)  in tcp_metrics_fill_info()
820 	if (tm->tcpm_ts_stamp) {  in tcp_metrics_fill_info()
822 				(s32) (get_seconds() - tm->tcpm_ts_stamp)) < 0)  in tcp_metrics_fill_info()
825 				tm->tcpm_ts) < 0)  in tcp_metrics_fill_info()
836 			u32 val = tm->tcpm_vals[i];  in tcp_metrics_fill_info()
870 			tfom_copy[0] = tm->tcpm_fastopen;  in tcp_metrics_fill_info()
898 				 struct tcp_metrics_block *tm)  in tcp_metrics_dump_info()  argument
908 	if (tcp_metrics_fill_info(skb, tm) < 0)  in tcp_metrics_dump_info()
928 		struct tcp_metrics_block *tm;  in tcp_metrics_nl_dump()  local
932 		for (col = 0, tm = rcu_dereference(hb->chain); tm;  in tcp_metrics_nl_dump()
933 		     tm = rcu_dereference(tm->tcpm_next), col++) {  in tcp_metrics_nl_dump()
934 			if (!net_eq(tm_net(tm), net))  in tcp_metrics_nl_dump()
938 			if (tcp_metrics_dump_info(skb, cb, tm) < 0) {  in tcp_metrics_nl_dump()
995 	struct tcp_metrics_block *tm;  in tcp_metrics_nl_cmd_get()  local
1025 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in tcp_metrics_nl_cmd_get()
1026 	     tm = rcu_dereference(tm->tcpm_next)) {  in tcp_metrics_nl_cmd_get()
1027 		if (addr_same(&tm->tcpm_daddr, &daddr) &&  in tcp_metrics_nl_cmd_get()
1028 		    (!src || addr_same(&tm->tcpm_saddr, &saddr)) &&  in tcp_metrics_nl_cmd_get()
1029 		    net_eq(tm_net(tm), net)) {  in tcp_metrics_nl_cmd_get()
1030 			ret = tcp_metrics_fill_info(msg, tm);  in tcp_metrics_nl_cmd_get()
1053 	struct tcp_metrics_block *tm;  in tcp_metrics_flush_all()  local
1060 		for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) {  in tcp_metrics_flush_all()
1061 			if (net_eq(tm_net(tm), net)) {  in tcp_metrics_flush_all()
1062 				*pp = tm->tcpm_next;  in tcp_metrics_flush_all()
1063 				kfree_rcu(tm, rcu_head);  in tcp_metrics_flush_all()
1065 				pp = &tm->tcpm_next;  in tcp_metrics_flush_all()
1075 	struct tcp_metrics_block *tm;  in tcp_metrics_nl_cmd_del()  local
1099 	for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) {  in tcp_metrics_nl_cmd_del()
1100 		if (addr_same(&tm->tcpm_daddr, &daddr) &&  in tcp_metrics_nl_cmd_del()
1101 		    (!src || addr_same(&tm->tcpm_saddr, &saddr)) &&  in tcp_metrics_nl_cmd_del()
1102 		    net_eq(tm_net(tm), net)) {  in tcp_metrics_nl_cmd_del()
1103 			*pp = tm->tcpm_next;  in tcp_metrics_nl_cmd_del()
1104 			kfree_rcu(tm, rcu_head);  in tcp_metrics_nl_cmd_del()
1107 			pp = &tm->tcpm_next;  in tcp_metrics_nl_cmd_del()