569 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)  in fnhe_flush_routes()  argument
573 	rt = rcu_dereference(fnhe->fnhe_rth_input);  in fnhe_flush_routes()
575 		RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);  in fnhe_flush_routes()
578 	rt = rcu_dereference(fnhe->fnhe_rth_output);  in fnhe_flush_routes()
580 		RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);  in fnhe_flush_routes()
587 	struct fib_nh_exception *fnhe, *oldest;  in fnhe_oldest()  local
590 	for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;  in fnhe_oldest()
591 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {  in fnhe_oldest()
592 		if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))  in fnhe_oldest()
593 			oldest = fnhe;  in fnhe_oldest()
609 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)  in fill_route_from_fnhe()  argument
611 	rt->rt_pmtu = fnhe->fnhe_pmtu;  in fill_route_from_fnhe()
612 	rt->dst.expires = fnhe->fnhe_expires;  in fill_route_from_fnhe()
614 	if (fnhe->fnhe_gw) {  in fill_route_from_fnhe()
616 		rt->rt_gateway = fnhe->fnhe_gw;  in fill_route_from_fnhe()
625 	struct fib_nh_exception *fnhe;  in update_or_create_fnhe()  local
644 	for (fnhe = rcu_dereference(hash->chain); fnhe;  in update_or_create_fnhe()
645 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {  in update_or_create_fnhe()
646 		if (fnhe->fnhe_daddr == daddr)  in update_or_create_fnhe()
651 	if (fnhe) {  in update_or_create_fnhe()
653 			fnhe->fnhe_gw = gw;  in update_or_create_fnhe()
655 			fnhe->fnhe_pmtu = pmtu;  in update_or_create_fnhe()
656 			fnhe->fnhe_expires = max(1UL, expires);  in update_or_create_fnhe()
659 		rt = rcu_dereference(fnhe->fnhe_rth_input);  in update_or_create_fnhe()
661 			fill_route_from_fnhe(rt, fnhe);  in update_or_create_fnhe()
662 		rt = rcu_dereference(fnhe->fnhe_rth_output);  in update_or_create_fnhe()
664 			fill_route_from_fnhe(rt, fnhe);  in update_or_create_fnhe()
667 			fnhe = fnhe_oldest(hash);  in update_or_create_fnhe()
669 			fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);  in update_or_create_fnhe()
670 			if (!fnhe)  in update_or_create_fnhe()
673 			fnhe->fnhe_next = hash->chain;  in update_or_create_fnhe()
674 			rcu_assign_pointer(hash->chain, fnhe);  in update_or_create_fnhe()
676 		fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));  in update_or_create_fnhe()
677 		fnhe->fnhe_daddr = daddr;  in update_or_create_fnhe()
678 		fnhe->fnhe_gw = gw;  in update_or_create_fnhe()
679 		fnhe->fnhe_pmtu = pmtu;  in update_or_create_fnhe()
680 		fnhe->fnhe_expires = expires;  in update_or_create_fnhe()
699 	fnhe->fnhe_stamp = jiffies;  in update_or_create_fnhe()
1255 	struct fib_nh_exception *fnhe;  in find_exception()  local
1263 	for (fnhe = rcu_dereference(hash[hval].chain); fnhe;  in find_exception()
1264 	     fnhe = rcu_dereference(fnhe->fnhe_next)) {  in find_exception()
1265 		if (fnhe->fnhe_daddr == daddr)  in find_exception()
1266 			return fnhe;  in find_exception()
1271 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,  in rt_bind_exception()  argument
1278 	if (daddr == fnhe->fnhe_daddr) {  in rt_bind_exception()
1284 			porig = &fnhe->fnhe_rth_input;  in rt_bind_exception()
1286 			porig = &fnhe->fnhe_rth_output;  in rt_bind_exception()
1289 		if (fnhe->fnhe_genid != genid) {  in rt_bind_exception()
1290 			fnhe->fnhe_genid = genid;  in rt_bind_exception()
1291 			fnhe->fnhe_gw = 0;  in rt_bind_exception()
1292 			fnhe->fnhe_pmtu = 0;  in rt_bind_exception()
1293 			fnhe->fnhe_expires = 0;  in rt_bind_exception()
1294 			fnhe_flush_routes(fnhe);  in rt_bind_exception()
1297 		fill_route_from_fnhe(rt, fnhe);  in rt_bind_exception()
1308 		fnhe->fnhe_stamp = jiffies;  in rt_bind_exception()
1398 			   struct fib_nh_exception *fnhe,  in rt_set_nexthop()  argument
1415 		if (unlikely(fnhe))  in rt_set_nexthop()
1416 			cached = rt_bind_exception(rt, fnhe, daddr);  in rt_set_nexthop()
1563 	struct fib_nh_exception *fnhe, __rcu **fnhe_p;  in ip_del_fnhe()  local
1573 	fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));  in ip_del_fnhe()
1574 	while (fnhe) {  in ip_del_fnhe()
1575 		if (fnhe->fnhe_daddr == daddr) {  in ip_del_fnhe()
1577 				fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));  in ip_del_fnhe()
1578 			fnhe_flush_routes(fnhe);  in ip_del_fnhe()
1579 			kfree_rcu(fnhe, rcu);  in ip_del_fnhe()
1582 		fnhe_p = &fnhe->fnhe_next;  in ip_del_fnhe()
1583 		fnhe = rcu_dereference_protected(fnhe->fnhe_next,  in ip_del_fnhe()
1596 	struct fib_nh_exception *fnhe;  in __mkroute_input()  local
1641 	fnhe = find_exception(&FIB_RES_NH(*res), daddr);  in __mkroute_input()
1643 		if (fnhe) {  in __mkroute_input()
1644 			rth = rcu_dereference(fnhe->fnhe_rth_input);  in __mkroute_input()
1648 				fnhe = NULL;  in __mkroute_input()
1678 	rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);  in __mkroute_input()
2007 	struct fib_nh_exception *fnhe;  in __mkroute_output()  local
2062 	fnhe = NULL;  in __mkroute_output()
2068 		fnhe = find_exception(nh, fl4->daddr);  in __mkroute_output()
2069 		if (fnhe) {  in __mkroute_output()
2070 			prth = &fnhe->fnhe_rth_output;  in __mkroute_output()
2075 				fnhe = NULL;  in __mkroute_output()
2129 	rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);  in __mkroute_output()