Lines Matching refs:po
189 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
193 static void *packet_previous_frame(struct packet_sock *po,
243 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
244 static void __fanout_link(struct sock *sk, struct packet_sock *po);
283 static struct net_device *packet_cached_dev_get(struct packet_sock *po) in packet_cached_dev_get() argument
288 dev = rcu_dereference(po->cached_dev); in packet_cached_dev_get()
296 static void packet_cached_dev_assign(struct packet_sock *po, in packet_cached_dev_assign() argument
299 rcu_assign_pointer(po->cached_dev, dev); in packet_cached_dev_assign()
302 static void packet_cached_dev_reset(struct packet_sock *po) in packet_cached_dev_reset() argument
304 RCU_INIT_POINTER(po->cached_dev, NULL); in packet_cached_dev_reset()
307 static bool packet_use_direct_xmit(const struct packet_sock *po) in packet_use_direct_xmit() argument
309 return po->xmit == packet_direct_xmit; in packet_use_direct_xmit()
339 struct packet_sock *po = pkt_sk(sk); in register_prot_hook() local
341 if (!po->running) { in register_prot_hook()
342 if (po->fanout) in register_prot_hook()
343 __fanout_link(sk, po); in register_prot_hook()
345 dev_add_pack(&po->prot_hook); in register_prot_hook()
348 po->running = 1; in register_prot_hook()
361 struct packet_sock *po = pkt_sk(sk); in __unregister_prot_hook() local
363 po->running = 0; in __unregister_prot_hook()
365 if (po->fanout) in __unregister_prot_hook()
366 __fanout_unlink(sk, po); in __unregister_prot_hook()
368 __dev_remove_pack(&po->prot_hook); in __unregister_prot_hook()
373 spin_unlock(&po->bind_lock); in __unregister_prot_hook()
375 spin_lock(&po->bind_lock); in __unregister_prot_hook()
381 struct packet_sock *po = pkt_sk(sk); in unregister_prot_hook() local
383 if (po->running) in unregister_prot_hook()
394 static void __packet_set_status(struct packet_sock *po, void *frame, int status) in __packet_set_status() argument
399 switch (po->tp_version) { in __packet_set_status()
417 static int __packet_get_status(struct packet_sock *po, void *frame) in __packet_get_status() argument
424 switch (po->tp_version) { in __packet_get_status()
455 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame, in __packet_set_timestamp() argument
462 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in __packet_set_timestamp()
466 switch (po->tp_version) { in __packet_set_timestamp()
488 static void *packet_lookup_frame(struct packet_sock *po, in packet_lookup_frame() argument
502 if (status != __packet_get_status(po, h.raw)) in packet_lookup_frame()
508 static void *packet_current_frame(struct packet_sock *po, in packet_current_frame() argument
512 return packet_lookup_frame(po, rb, rb->head, status); in packet_current_frame()
520 static void prb_shutdown_retire_blk_timer(struct packet_sock *po, in prb_shutdown_retire_blk_timer() argument
526 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) : in prb_shutdown_retire_blk_timer()
527 GET_PBDQC_FROM_RB(&po->rx_ring); in prb_shutdown_retire_blk_timer()
536 static void prb_init_blk_timer(struct packet_sock *po, in prb_init_blk_timer() argument
541 pkc->retire_blk_timer.data = (long)po; in prb_init_blk_timer()
546 static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring) in prb_setup_retire_blk_timer() argument
553 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) : in prb_setup_retire_blk_timer()
554 GET_PBDQC_FROM_RB(&po->rx_ring); in prb_setup_retire_blk_timer()
555 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired); in prb_setup_retire_blk_timer()
558 static int prb_calc_retire_blk_tmo(struct packet_sock *po, in prb_calc_retire_blk_tmo() argument
568 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex); in prb_calc_retire_blk_tmo()
607 static void init_prb_bdqc(struct packet_sock *po, in init_prb_bdqc() argument
623 p1->hdrlen = po->tp_hdrlen; in init_prb_bdqc()
624 p1->version = po->tp_version; in init_prb_bdqc()
626 po->stats.stats3.tp_freeze_q_cnt = 0; in init_prb_bdqc()
630 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po, in init_prb_bdqc()
637 prb_setup_retire_blk_timer(po, tx_ring); in init_prb_bdqc()
676 struct packet_sock *po = (struct packet_sock *)data; in prb_retire_rx_blk_timer_expired() local
677 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in prb_retire_rx_blk_timer_expired()
681 spin_lock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
711 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO); in prb_retire_rx_blk_timer_expired()
712 if (!prb_dispatch_next_block(pkc, po)) in prb_retire_rx_blk_timer_expired()
744 spin_unlock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
792 struct packet_sock *po, unsigned int stat) in prb_close_block() argument
798 struct sock *sk = &po->sk; in prb_close_block()
800 if (po->stats.stats3.tp_drops) in prb_close_block()
905 struct packet_sock *po) in prb_freeze_queue() argument
908 po->stats.stats3.tp_freeze_q_cnt++; in prb_freeze_queue()
920 struct packet_sock *po) in prb_dispatch_next_block() argument
931 prb_freeze_queue(pkc, po); in prb_dispatch_next_block()
945 struct packet_sock *po, unsigned int status) in prb_retire_current_block() argument
966 prb_close_block(pkc, pbd, po, status); in prb_retire_current_block()
1044 static void *__packet_lookup_frame_in_block(struct packet_sock *po, in __packet_lookup_frame_in_block() argument
1054 pkc = GET_PBDQC_FROM_RB(&po->rx_ring); in __packet_lookup_frame_in_block()
1089 prb_retire_current_block(pkc, po, 0); in __packet_lookup_frame_in_block()
1092 curr = (char *)prb_dispatch_next_block(pkc, po); in __packet_lookup_frame_in_block()
1106 static void *packet_current_rx_frame(struct packet_sock *po, in packet_current_rx_frame() argument
1111 switch (po->tp_version) { in packet_current_rx_frame()
1114 curr = packet_lookup_frame(po, &po->rx_ring, in packet_current_rx_frame()
1115 po->rx_ring.head, status); in packet_current_rx_frame()
1118 return __packet_lookup_frame_in_block(po, skb, status, len); in packet_current_rx_frame()
1126 static void *prb_lookup_block(struct packet_sock *po, in prb_lookup_block() argument
1150 static void *__prb_previous_block(struct packet_sock *po, in __prb_previous_block() argument
1155 return prb_lookup_block(po, rb, previous, status); in __prb_previous_block()
1158 static void *packet_previous_rx_frame(struct packet_sock *po, in packet_previous_rx_frame() argument
1162 if (po->tp_version <= TPACKET_V2) in packet_previous_rx_frame()
1163 return packet_previous_frame(po, rb, status); in packet_previous_rx_frame()
1165 return __prb_previous_block(po, rb, status); in packet_previous_rx_frame()
1168 static void packet_increment_rx_head(struct packet_sock *po, in packet_increment_rx_head() argument
1171 switch (po->tp_version) { in packet_increment_rx_head()
1183 static void *packet_previous_frame(struct packet_sock *po, in packet_previous_frame() argument
1188 return packet_lookup_frame(po, rb, previous, status); in packet_previous_frame()
1221 static int packet_alloc_pending(struct packet_sock *po) in packet_alloc_pending() argument
1223 po->rx_ring.pending_refcnt = NULL; in packet_alloc_pending()
1225 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int); in packet_alloc_pending()
1226 if (unlikely(po->tx_ring.pending_refcnt == NULL)) in packet_alloc_pending()
1232 static void packet_free_pending(struct packet_sock *po) in packet_free_pending() argument
1234 free_percpu(po->tx_ring.pending_refcnt); in packet_free_pending()
1237 static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) in packet_rcv_has_room() argument
1239 struct sock *sk = &po->sk; in packet_rcv_has_room()
1242 if (po->prot_hook.func != tpacket_rcv) in packet_rcv_has_room()
1247 if (po->tp_version == TPACKET_V3) in packet_rcv_has_room()
1248 has_room = prb_lookup_block(po, &po->rx_ring, in packet_rcv_has_room()
1249 po->rx_ring.prb_bdqc.kactive_blk_num, in packet_rcv_has_room()
1252 has_room = packet_lookup_frame(po, &po->rx_ring, in packet_rcv_has_room()
1253 po->rx_ring.head, in packet_rcv_has_room()
1343 struct packet_sock *po; in packet_rcv_fanout() local
1379 po = pkt_sk(f->arr[idx]); in packet_rcv_fanout()
1381 unlikely(!packet_rcv_has_room(po, skb))) { in packet_rcv_fanout()
1383 po = pkt_sk(f->arr[idx]); in packet_rcv_fanout()
1386 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); in packet_rcv_fanout()
1393 static void __fanout_link(struct sock *sk, struct packet_sock *po) in __fanout_link() argument
1395 struct packet_fanout *f = po->fanout; in __fanout_link()
1404 static void __fanout_unlink(struct sock *sk, struct packet_sock *po) in __fanout_unlink() argument
1406 struct packet_fanout *f = po->fanout; in __fanout_unlink()
1430 struct packet_sock *po = pkt_sk(sk); in fanout_add() local
1450 if (!po->running) in fanout_add()
1453 if (po->fanout) in fanout_add()
1481 match->prot_hook.type = po->prot_hook.type; in fanout_add()
1482 match->prot_hook.dev = po->prot_hook.dev; in fanout_add()
1491 match->prot_hook.type == po->prot_hook.type && in fanout_add()
1492 match->prot_hook.dev == po->prot_hook.dev) { in fanout_add()
1495 __dev_remove_pack(&po->prot_hook); in fanout_add()
1496 po->fanout = match; in fanout_add()
1498 __fanout_link(sk, po); in fanout_add()
1509 struct packet_sock *po = pkt_sk(sk); in fanout_release() local
1512 f = po->fanout; in fanout_release()
1517 po->fanout = NULL; in fanout_release()
1763 struct packet_sock *po; in packet_rcv() local
1772 po = pkt_sk(sk); in packet_rcv()
1824 if (unlikely(po->origdev)) in packet_rcv()
1847 po->stats.stats1.tp_packets++; in packet_rcv()
1856 po->stats.stats1.tp_drops++; in packet_rcv()
1874 struct packet_sock *po; in tpacket_rcv() local
1897 po = pkt_sk(sk); in tpacket_rcv()
1928 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + in tpacket_rcv()
1929 po->tp_reserve; in tpacket_rcv()
1932 netoff = TPACKET_ALIGN(po->tp_hdrlen + in tpacket_rcv()
1934 po->tp_reserve; in tpacket_rcv()
1937 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
1938 if (macoff + snaplen > po->rx_ring.frame_size) { in tpacket_rcv()
1939 if (po->copy_thresh && in tpacket_rcv()
1950 snaplen = po->rx_ring.frame_size - macoff; in tpacket_rcv()
1955 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) { in tpacket_rcv()
1958 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff; in tpacket_rcv()
1964 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len; in tpacket_rcv()
1968 h.raw = packet_current_rx_frame(po, skb, in tpacket_rcv()
1972 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
1973 packet_increment_rx_head(po, &po->rx_ring); in tpacket_rcv()
1980 if (po->stats.stats1.tp_drops) in tpacket_rcv()
1983 po->stats.stats1.tp_packets++; in tpacket_rcv()
1992 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp))) in tpacket_rcv()
1997 switch (po->tp_version) { in tpacket_rcv()
2049 if (unlikely(po->origdev)) in tpacket_rcv()
2057 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2069 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
2070 __packet_set_status(po, h.raw, status); in tpacket_rcv()
2073 prb_clear_blk_fill_status(&po->rx_ring); in tpacket_rcv()
2086 po->stats.stats1.tp_drops++; in tpacket_rcv()
2096 struct packet_sock *po = pkt_sk(skb->sk); in tpacket_destruct_skb() local
2098 if (likely(po->tx_ring.pg_vec)) { in tpacket_destruct_skb()
2103 packet_dec_pending(&po->tx_ring); in tpacket_destruct_skb()
2105 ts = __packet_set_timestamp(po, ph, skb); in tpacket_destruct_skb()
2106 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); in tpacket_destruct_skb()
2133 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, in tpacket_fill_skb() argument
2139 struct socket *sock = po->sk.sk_socket; in tpacket_fill_skb()
2148 skb->priority = po->sk.sk_priority; in tpacket_fill_skb()
2149 skb->mark = po->sk.sk_mark; in tpacket_fill_skb()
2150 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags); in tpacket_fill_skb()
2153 switch (po->tp_version) { in tpacket_fill_skb()
2169 if (unlikely(po->tp_tx_has_off)) { in tpacket_fill_skb()
2171 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_fill_skb()
2172 off_max = po->tx_ring.frame_size - tp_len; in tpacket_fill_skb()
2174 switch (po->tp_version) { in tpacket_fill_skb()
2183 switch (po->tp_version) { in tpacket_fill_skb()
2196 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_fill_skb()
2228 atomic_add(to_write, &po->sk.sk_wmem_alloc); in tpacket_fill_skb()
2255 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) in tpacket_snd() argument
2270 mutex_lock(&po->pg_vec_lock); in tpacket_snd()
2273 dev = packet_cached_dev_get(po); in tpacket_snd()
2274 proto = po->num; in tpacket_snd()
2286 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex); in tpacket_snd()
2296 if (po->sk.sk_socket->type == SOCK_RAW) in tpacket_snd()
2298 size_max = po->tx_ring.frame_size in tpacket_snd()
2299 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); in tpacket_snd()
2305 ph = packet_current_frame(po, &po->tx_ring, in tpacket_snd()
2316 skb = sock_alloc_send_skb(&po->sk, in tpacket_snd()
2326 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto, in tpacket_snd()
2334 if (po->tp_loss) { in tpacket_snd()
2335 __packet_set_status(po, ph, in tpacket_snd()
2337 packet_increment_head(&po->tx_ring); in tpacket_snd()
2350 __packet_set_status(po, ph, TP_STATUS_SENDING); in tpacket_snd()
2351 packet_inc_pending(&po->tx_ring); in tpacket_snd()
2354 err = po->xmit(skb); in tpacket_snd()
2357 if (err && __packet_get_status(po, ph) == in tpacket_snd()
2369 packet_increment_head(&po->tx_ring); in tpacket_snd()
2378 (need_wait && packet_read_pending(&po->tx_ring)))); in tpacket_snd()
2384 __packet_set_status(po, ph, status); in tpacket_snd()
2389 mutex_unlock(&po->pg_vec_lock); in tpacket_snd()
2429 struct packet_sock *po = pkt_sk(sk); in packet_snd() local
2440 dev = packet_cached_dev_get(po); in packet_snd()
2441 proto = po->num; in packet_snd()
2463 if (po->has_vnet_hdr) { in packet_snd()
2566 if (po->has_vnet_hdr) { in packet_snd()
2592 err = po->xmit(skb); in packet_snd()
2612 struct packet_sock *po = pkt_sk(sk); in packet_sendmsg() local
2614 if (po->tx_ring.pg_vec) in packet_sendmsg()
2615 return tpacket_snd(po, msg); in packet_sendmsg()
2628 struct packet_sock *po; in packet_release() local
2636 po = pkt_sk(sk); in packet_release()
2646 spin_lock(&po->bind_lock); in packet_release()
2648 packet_cached_dev_reset(po); in packet_release()
2650 if (po->prot_hook.dev) { in packet_release()
2651 dev_put(po->prot_hook.dev); in packet_release()
2652 po->prot_hook.dev = NULL; in packet_release()
2654 spin_unlock(&po->bind_lock); in packet_release()
2658 if (po->rx_ring.pg_vec) { in packet_release()
2663 if (po->tx_ring.pg_vec) { in packet_release()
2680 packet_free_pending(po); in packet_release()
2694 struct packet_sock *po = pkt_sk(sk); in packet_do_bind() local
2702 if (po->fanout) in packet_do_bind()
2706 spin_lock(&po->bind_lock); in packet_do_bind()
2726 proto_curr = po->prot_hook.type; in packet_do_bind()
2727 dev_curr = po->prot_hook.dev; in packet_do_bind()
2732 if (po->running) { in packet_do_bind()
2736 dev_curr = po->prot_hook.dev; in packet_do_bind()
2742 po->num = proto; in packet_do_bind()
2743 po->prot_hook.type = proto; in packet_do_bind()
2747 po->prot_hook.dev = NULL; in packet_do_bind()
2748 po->ifindex = -1; in packet_do_bind()
2749 packet_cached_dev_reset(po); in packet_do_bind()
2751 po->prot_hook.dev = dev; in packet_do_bind()
2752 po->ifindex = dev ? dev->ifindex : 0; in packet_do_bind()
2753 packet_cached_dev_assign(po, dev); in packet_do_bind()
2772 spin_unlock(&po->bind_lock); in packet_do_bind()
2830 struct packet_sock *po; in packet_create() local
2853 po = pkt_sk(sk); in packet_create()
2855 po->num = proto; in packet_create()
2856 po->xmit = dev_queue_xmit; in packet_create()
2858 err = packet_alloc_pending(po); in packet_create()
2862 packet_cached_dev_reset(po); in packet_create()
2871 spin_lock_init(&po->bind_lock); in packet_create()
2872 mutex_init(&po->pg_vec_lock); in packet_create()
2873 po->prot_hook.func = packet_rcv; in packet_create()
2876 po->prot_hook.func = packet_rcv_spkt; in packet_create()
2878 po->prot_hook.af_packet_priv = sk; in packet_create()
2881 po->prot_hook.type = proto; in packet_create()
3104 struct packet_sock *po = pkt_sk(sk); in packet_getname() local
3111 sll->sll_ifindex = po->ifindex; in packet_getname()
3112 sll->sll_protocol = po->num; in packet_getname()
3115 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex); in packet_getname()
3177 struct packet_sock *po = pkt_sk(sk); in packet_mc_add() local
3199 for (ml = po->mclist; ml; ml = ml->next) { in packet_mc_add()
3216 i->next = po->mclist; in packet_mc_add()
3217 po->mclist = i; in packet_mc_add()
3220 po->mclist = i->next; in packet_mc_add()
3257 struct packet_sock *po = pkt_sk(sk); in packet_flush_mclist() local
3260 if (!po->mclist) in packet_flush_mclist()
3264 while ((ml = po->mclist) != NULL) { in packet_flush_mclist()
3267 po->mclist = ml->next; in packet_flush_mclist()
3280 struct packet_sock *po = pkt_sk(sk); in packet_setsockopt() local
3314 switch (po->tp_version) { in packet_setsockopt()
3351 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3359 po->tp_version = val; in packet_setsockopt()
3371 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3375 po->tp_reserve = val; in packet_setsockopt()
3384 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3388 po->tp_loss = !!val; in packet_setsockopt()
3400 po->auxdata = !!val; in packet_setsockopt()
3412 po->origdev = !!val; in packet_setsockopt()
3421 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3428 po->has_vnet_hdr = !!val; in packet_setsockopt()
3440 po->tp_tstamp = val; in packet_setsockopt()
3460 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3464 po->tp_tx_has_off = !!val; in packet_setsockopt()
3476 po->xmit = val ? packet_direct_xmit : dev_queue_xmit; in packet_setsockopt()
3490 struct packet_sock *po = pkt_sk(sk); in packet_getsockopt() local
3506 memcpy(&st, &po->stats, sizeof(st)); in packet_getsockopt()
3507 memset(&po->stats, 0, sizeof(po->stats)); in packet_getsockopt()
3510 if (po->tp_version == TPACKET_V3) { in packet_getsockopt()
3522 val = po->auxdata; in packet_getsockopt()
3525 val = po->origdev; in packet_getsockopt()
3528 val = po->has_vnet_hdr; in packet_getsockopt()
3531 val = po->tp_version; in packet_getsockopt()
3553 val = po->tp_reserve; in packet_getsockopt()
3556 val = po->tp_loss; in packet_getsockopt()
3559 val = po->tp_tstamp; in packet_getsockopt()
3562 val = (po->fanout ? in packet_getsockopt()
3563 ((u32)po->fanout->id | in packet_getsockopt()
3564 ((u32)po->fanout->type << 16) | in packet_getsockopt()
3565 ((u32)po->fanout->flags << 24)) : in packet_getsockopt()
3569 val = po->tp_tx_has_off; in packet_getsockopt()
3572 val = packet_use_direct_xmit(po); in packet_getsockopt()
3597 struct packet_sock *po = pkt_sk(sk); in packet_notifier() local
3601 if (po->mclist) in packet_notifier()
3602 packet_dev_mclist_delete(dev, &po->mclist); in packet_notifier()
3606 if (dev->ifindex == po->ifindex) { in packet_notifier()
3607 spin_lock(&po->bind_lock); in packet_notifier()
3608 if (po->running) { in packet_notifier()
3615 packet_cached_dev_reset(po); in packet_notifier()
3616 po->ifindex = -1; in packet_notifier()
3617 if (po->prot_hook.dev) in packet_notifier()
3618 dev_put(po->prot_hook.dev); in packet_notifier()
3619 po->prot_hook.dev = NULL; in packet_notifier()
3621 spin_unlock(&po->bind_lock); in packet_notifier()
3625 if (dev->ifindex == po->ifindex) { in packet_notifier()
3626 spin_lock(&po->bind_lock); in packet_notifier()
3627 if (po->num) in packet_notifier()
3629 spin_unlock(&po->bind_lock); in packet_notifier()
3696 struct packet_sock *po = pkt_sk(sk); in packet_poll() local
3700 if (po->rx_ring.pg_vec) { in packet_poll()
3701 if (!packet_previous_rx_frame(po, &po->rx_ring, in packet_poll()
3707 if (po->tx_ring.pg_vec) { in packet_poll()
3708 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) in packet_poll()
3817 struct packet_sock *po = pkt_sk(sk); in packet_set_ring() local
3827 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
3832 rb = tx_ring ? &po->tx_ring : &po->rx_ring; in packet_set_ring()
3837 if (atomic_read(&po->mapped)) in packet_set_ring()
3849 switch (po->tp_version) { in packet_set_ring()
3851 po->tp_hdrlen = TPACKET_HDRLEN; in packet_set_ring()
3854 po->tp_hdrlen = TPACKET2_HDRLEN; in packet_set_ring()
3857 po->tp_hdrlen = TPACKET3_HDRLEN; in packet_set_ring()
3866 if (po->tp_version >= TPACKET_V3 && in packet_set_ring()
3870 if (unlikely(req->tp_frame_size < po->tp_hdrlen + in packet_set_ring()
3871 po->tp_reserve)) in packet_set_ring()
3888 switch (po->tp_version) { in packet_set_ring()
3894 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring); in packet_set_ring()
3910 spin_lock(&po->bind_lock); in packet_set_ring()
3911 was_running = po->running; in packet_set_ring()
3912 num = po->num; in packet_set_ring()
3914 po->num = 0; in packet_set_ring()
3917 spin_unlock(&po->bind_lock); in packet_set_ring()
3922 mutex_lock(&po->pg_vec_lock); in packet_set_ring()
3923 if (closing || atomic_read(&po->mapped) == 0) { in packet_set_ring()
3936 po->prot_hook.func = (po->rx_ring.pg_vec) ? in packet_set_ring()
3939 if (atomic_read(&po->mapped)) in packet_set_ring()
3941 atomic_read(&po->mapped)); in packet_set_ring()
3943 mutex_unlock(&po->pg_vec_lock); in packet_set_ring()
3945 spin_lock(&po->bind_lock); in packet_set_ring()
3947 po->num = num; in packet_set_ring()
3950 spin_unlock(&po->bind_lock); in packet_set_ring()
3951 if (closing && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
3954 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue); in packet_set_ring()
3968 struct packet_sock *po = pkt_sk(sk); in packet_mmap() local
3978 mutex_lock(&po->pg_vec_lock); in packet_mmap()
3981 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
3997 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
4017 atomic_inc(&po->mapped); in packet_mmap()
4022 mutex_unlock(&po->pg_vec_lock); in packet_mmap()
4107 const struct packet_sock *po = pkt_sk(s); in packet_seq_show() local
4114 ntohs(po->num), in packet_seq_show()
4115 po->ifindex, in packet_seq_show()
4116 po->running, in packet_seq_show()