58 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
59 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
248 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);  in xfrm_policy_flo_get()  local
250 	if (unlikely(pol->walk.dead))  in xfrm_policy_flo_get()
253 		xfrm_pol_hold(pol);  in xfrm_policy_flo_get()
260 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);  in xfrm_policy_flo_check()  local
262 	return !pol->walk.dead;  in xfrm_policy_flo_check()
415 	struct xfrm_policy *pol;  in xfrm_dst_hash_transfer()  local
421 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {  in xfrm_dst_hash_transfer()
424 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);  in xfrm_dst_hash_transfer()
425 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,  in xfrm_dst_hash_transfer()
426 				pol->family, nhashmask, dbits, sbits);  in xfrm_dst_hash_transfer()
428 			hlist_del(&pol->bydst);  in xfrm_dst_hash_transfer()
429 			hlist_add_head(&pol->bydst, ndsttable+h);  in xfrm_dst_hash_transfer()
434 			hlist_del(&pol->bydst);  in xfrm_dst_hash_transfer()
435 			hlist_add_behind(&pol->bydst, entry0);  in xfrm_dst_hash_transfer()
437 		entry0 = &pol->bydst;  in xfrm_dst_hash_transfer()
450 	struct xfrm_policy *pol;  in xfrm_idx_hash_transfer()  local
452 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {  in xfrm_idx_hash_transfer()
455 		h = __idx_hash(pol->index, nhashmask);  in xfrm_idx_hash_transfer()
456 		hlist_add_head(&pol->byidx, nidxtable+h);  in xfrm_idx_hash_transfer()
580 	struct xfrm_policy *pol;  in xfrm_hash_rebuild()  local
632 		hlist_for_each_entry(pol, chain, bydst) {  in xfrm_hash_rebuild()
633 			if (policy->priority >= pol->priority)  in xfrm_hash_rebuild()
634 				newpos = &pol->bydst;  in xfrm_hash_rebuild()
733 				   struct xfrm_policy *pol)  in xfrm_policy_mark_match()
737 	if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)  in xfrm_policy_mark_match()
740 	if ((mark & pol->mark.m) == pol->mark.v &&  in xfrm_policy_mark_match()
741 	    policy->priority == pol->priority)  in xfrm_policy_mark_match()
750 	struct xfrm_policy *pol;  in xfrm_policy_insert()  local
759 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_insert()
760 		if (pol->type == policy->type &&  in xfrm_policy_insert()
761 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert()
762 		    xfrm_policy_mark_match(policy, pol) &&  in xfrm_policy_insert()
763 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert()
769 			delpol = pol;  in xfrm_policy_insert()
770 			if (policy->priority > pol->priority)  in xfrm_policy_insert()
772 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert()
773 			newpos = &pol->bydst;  in xfrm_policy_insert()
818 	struct xfrm_policy *pol, *ret;  in xfrm_policy_bysel_ctx()  local
825 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_bysel_ctx()
826 		if (pol->type == type &&  in xfrm_policy_bysel_ctx()
827 		    (mark & pol->mark.m) == pol->mark.v &&  in xfrm_policy_bysel_ctx()
828 		    !selector_cmp(sel, &pol->selector) &&  in xfrm_policy_bysel_ctx()
829 		    xfrm_sec_ctx_match(ctx, pol->security)) {  in xfrm_policy_bysel_ctx()
830 			xfrm_pol_hold(pol);  in xfrm_policy_bysel_ctx()
833 								pol->security);  in xfrm_policy_bysel_ctx()
836 					return pol;  in xfrm_policy_bysel_ctx()
838 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_bysel_ctx()
840 			ret = pol;  in xfrm_policy_bysel_ctx()
855 	struct xfrm_policy *pol, *ret;  in xfrm_policy_byid()  local
866 	hlist_for_each_entry(pol, chain, byidx) {  in xfrm_policy_byid()
867 		if (pol->type == type && pol->index == id &&  in xfrm_policy_byid()
868 		    (mark & pol->mark.m) == pol->mark.v) {  in xfrm_policy_byid()
869 			xfrm_pol_hold(pol);  in xfrm_policy_byid()
872 								pol->security);  in xfrm_policy_byid()
875 					return pol;  in xfrm_policy_byid()
877 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_byid()
879 			ret = pol;  in xfrm_policy_byid()
898 		struct xfrm_policy *pol;  in xfrm_policy_flush_secctx_check()  local
901 		hlist_for_each_entry(pol,  in xfrm_policy_flush_secctx_check()
903 			if (pol->type != type)  in xfrm_policy_flush_secctx_check()
905 			err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_flush_secctx_check()
907 				xfrm_audit_policy_delete(pol, 0, task_valid);  in xfrm_policy_flush_secctx_check()
912 			hlist_for_each_entry(pol,  in xfrm_policy_flush_secctx_check()
915 				if (pol->type != type)  in xfrm_policy_flush_secctx_check()
918 								pol->security);  in xfrm_policy_flush_secctx_check()
920 					xfrm_audit_policy_delete(pol, 0,  in xfrm_policy_flush_secctx_check()
948 		struct xfrm_policy *pol;  in xfrm_policy_flush()  local
952 		hlist_for_each_entry(pol,  in xfrm_policy_flush()
954 			if (pol->type != type)  in xfrm_policy_flush()
956 			__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
960 			xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
962 			xfrm_policy_kill(pol);  in xfrm_policy_flush()
970 			hlist_for_each_entry(pol,  in xfrm_policy_flush()
973 				if (pol->type != type)  in xfrm_policy_flush()
975 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
979 				xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
980 				xfrm_policy_kill(pol);  in xfrm_policy_flush()
1000 	struct xfrm_policy *pol;  in xfrm_policy_walk()  local
1019 		pol = container_of(x, struct xfrm_policy, walk);  in xfrm_policy_walk()
1021 		    walk->type != pol->type)  in xfrm_policy_walk()
1023 		error = func(pol, xfrm_policy_id2dir(pol->index),  in xfrm_policy_walk()
1067 static int xfrm_policy_match(const struct xfrm_policy *pol,  in xfrm_policy_match()  argument
1071 	const struct xfrm_selector *sel = &pol->selector;  in xfrm_policy_match()
1075 	if (pol->family != family ||  in xfrm_policy_match()
1076 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||  in xfrm_policy_match()
1077 	    pol->type != type)  in xfrm_policy_match()
1082 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,  in xfrm_policy_match()
1093 	struct xfrm_policy *pol, *ret;  in xfrm_policy_lookup_bytype()  local
1106 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
1107 		err = xfrm_policy_match(pol, fl, type, family, dir);  in xfrm_policy_lookup_bytype()
1116 			ret = pol;  in xfrm_policy_lookup_bytype()
1122 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
1123 		err = xfrm_policy_match(pol, fl, type, family, dir);  in xfrm_policy_lookup_bytype()
1131 		} else if (pol->priority < priority) {  in xfrm_policy_lookup_bytype()
1132 			ret = pol;  in xfrm_policy_lookup_bytype()
1148 	struct xfrm_policy *pol;  in __xfrm_policy_lookup()  local
1150 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);  in __xfrm_policy_lookup()
1151 	if (pol != NULL)  in __xfrm_policy_lookup()
1152 		return pol;  in __xfrm_policy_lookup()
1179 	struct xfrm_policy *pol;  in xfrm_policy_lookup()  local
1184 	pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));  in xfrm_policy_lookup()
1185 	if (IS_ERR_OR_NULL(pol))  in xfrm_policy_lookup()
1186 		return ERR_CAST(pol);  in xfrm_policy_lookup()
1190 	xfrm_pol_hold(pol);  in xfrm_policy_lookup()
1192 	return &pol->flo;  in xfrm_policy_lookup()
1215 	struct xfrm_policy *pol;  in xfrm_sk_policy_lookup()  local
1219 	if ((pol = sk->sk_policy[dir]) != NULL) {  in xfrm_sk_policy_lookup()
1220 		bool match = xfrm_selector_match(&pol->selector, fl,  in xfrm_sk_policy_lookup()
1225 			if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {  in xfrm_sk_policy_lookup()
1226 				pol = NULL;  in xfrm_sk_policy_lookup()
1229 			err = security_xfrm_policy_lookup(pol->security,  in xfrm_sk_policy_lookup()
1233 				xfrm_pol_hold(pol);  in xfrm_sk_policy_lookup()
1235 				pol = NULL;  in xfrm_sk_policy_lookup()
1237 				pol = ERR_PTR(err);  in xfrm_sk_policy_lookup()
1239 			pol = NULL;  in xfrm_sk_policy_lookup()
1243 	return pol;  in xfrm_sk_policy_lookup()
1246 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)  in __xfrm_policy_link()  argument
1248 	struct net *net = xp_net(pol);  in __xfrm_policy_link()
1250 	list_add(&pol->walk.all, &net->xfrm.policy_all);  in __xfrm_policy_link()
1252 	xfrm_pol_hold(pol);  in __xfrm_policy_link()
1255 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,  in __xfrm_policy_unlink()  argument
1258 	struct net *net = xp_net(pol);  in __xfrm_policy_unlink()
1260 	if (list_empty(&pol->walk.all))  in __xfrm_policy_unlink()
1264 	if (!hlist_unhashed(&pol->bydst)) {  in __xfrm_policy_unlink()
1265 		hlist_del(&pol->bydst);  in __xfrm_policy_unlink()
1266 		hlist_del(&pol->byidx);  in __xfrm_policy_unlink()
1269 	list_del_init(&pol->walk.all);  in __xfrm_policy_unlink()
1272 	return pol;  in __xfrm_policy_unlink()
1275 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_link()  argument
1277 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_link()
1280 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_unlink()  argument
1282 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_unlink()
1285 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)  in xfrm_policy_delete()  argument
1287 	struct net *net = xp_net(pol);  in xfrm_policy_delete()
1290 	pol = __xfrm_policy_unlink(pol, dir);  in xfrm_policy_delete()
1292 	if (pol) {  in xfrm_policy_delete()
1293 		xfrm_policy_kill(pol);  in xfrm_policy_delete()
1300 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)  in xfrm_sk_policy_insert()  argument
1302 	struct net *net = xp_net(pol);  in xfrm_sk_policy_insert()
1306 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)  in xfrm_sk_policy_insert()
1312 	sk->sk_policy[dir] = pol;  in xfrm_sk_policy_insert()
1313 	if (pol) {  in xfrm_sk_policy_insert()
1314 		pol->curlft.add_time = get_seconds();  in xfrm_sk_policy_insert()
1315 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);  in xfrm_sk_policy_insert()
1316 		xfrm_sk_policy_link(pol, dir);  in xfrm_sk_policy_insert()
1319 		if (pol)  in xfrm_sk_policy_insert()
1320 			xfrm_policy_requeue(old_pol, pol);  in xfrm_sk_policy_insert()
1890 	struct xfrm_policy *pol = (struct xfrm_policy *)arg;  in xfrm_policy_queue_process()  local
1891 	struct xfrm_policy_queue *pq = &pol->polq;  in xfrm_policy_queue_process()
1907 	dst = xfrm_lookup(xp_net(pol), dst->path, &fl,  in xfrm_policy_queue_process()
1920 			xfrm_pol_hold(pol);  in xfrm_policy_queue_process()
1938 		dst = xfrm_lookup(xp_net(pol), skb_dst(skb)->path,  in xfrm_policy_queue_process()
1953 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
1959 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
1967 	struct xfrm_policy *pol = xdst->pols[0];  in xdst_queue_output()  local
1968 	struct xfrm_policy_queue *pq = &pol->polq;  in xdst_queue_output()
1992 		xfrm_pol_put(pol);  in xdst_queue_output()
1997 		xfrm_pol_hold(pol);  in xdst_queue_output()
2447 	struct xfrm_policy *pol;  in __xfrm_policy_check()  local
2481 	pol = NULL;  in __xfrm_policy_check()
2483 		pol = xfrm_sk_policy_lookup(sk, dir, &fl);  in __xfrm_policy_check()
2484 		if (IS_ERR(pol)) {  in __xfrm_policy_check()
2490 	if (!pol) {  in __xfrm_policy_check()
2496 			pol = ERR_CAST(flo);  in __xfrm_policy_check()
2498 			pol = container_of(flo, struct xfrm_policy, flo);  in __xfrm_policy_check()
2501 	if (IS_ERR(pol)) {  in __xfrm_policy_check()
2506 	if (!pol) {  in __xfrm_policy_check()
2515 	pol->curlft.use_time = get_seconds();  in __xfrm_policy_check()
2517 	pols[0] = pol;  in __xfrm_policy_check()
2535 	if (pol->action == XFRM_POLICY_ALLOW) {  in __xfrm_policy_check()
2548 			if (pols[pi] != pol &&  in __xfrm_policy_check()
3158 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_migrate_policy_find()  local
3164 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
3165 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
3166 		    pol->type == type) {  in xfrm_migrate_policy_find()
3167 			ret = pol;  in xfrm_migrate_policy_find()
3173 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
3174 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
3175 		    pol->type == type &&  in xfrm_migrate_policy_find()
3176 		    pol->priority < priority) {  in xfrm_migrate_policy_find()
3177 			ret = pol;  in xfrm_migrate_policy_find()
3220 static int xfrm_policy_migrate(struct xfrm_policy *pol,  in xfrm_policy_migrate()  argument
3226 	write_lock_bh(&pol->lock);  in xfrm_policy_migrate()
3227 	if (unlikely(pol->walk.dead)) {  in xfrm_policy_migrate()
3229 		write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
3233 	for (i = 0; i < pol->xfrm_nr; i++) {  in xfrm_policy_migrate()
3235 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))  in xfrm_policy_migrate()
3238 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&  in xfrm_policy_migrate()
3239 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)  in xfrm_policy_migrate()
3242 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,  in xfrm_policy_migrate()
3243 			       sizeof(pol->xfrm_vec[i].id.daddr));  in xfrm_policy_migrate()
3244 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,  in xfrm_policy_migrate()
3245 			       sizeof(pol->xfrm_vec[i].saddr));  in xfrm_policy_migrate()
3246 			pol->xfrm_vec[i].encap_family = mp->new_family;  in xfrm_policy_migrate()
3248 			atomic_inc(&pol->genid);  in xfrm_policy_migrate()
3252 	write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
3299 	struct xfrm_policy *pol = NULL;  in xfrm_migrate()  local
3309 	if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {  in xfrm_migrate()
3330 	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)  in xfrm_migrate()
3342 	xfrm_pol_put(pol);  in xfrm_migrate()
3349 	if (pol)  in xfrm_migrate()
3350 		xfrm_pol_put(pol);  in xfrm_migrate()