1/*
2 * Common framework for low-level network console, dump, and debugger code
3 *
4 * Sep 8 2003  Matt Mackall <mpm@selenic.com>
5 *
6 * based on the netconsole code from:
7 *
8 * Copyright (C) 2001  Ingo Molnar <mingo@redhat.com>
9 * Copyright (C) 2002  Red Hat, Inc.
10 */
11
12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
14#include <linux/moduleparam.h>
15#include <linux/kernel.h>
16#include <linux/netdevice.h>
17#include <linux/etherdevice.h>
18#include <linux/string.h>
19#include <linux/if_arp.h>
20#include <linux/inetdevice.h>
21#include <linux/inet.h>
22#include <linux/interrupt.h>
23#include <linux/netpoll.h>
24#include <linux/sched.h>
25#include <linux/delay.h>
26#include <linux/rcupdate.h>
27#include <linux/workqueue.h>
28#include <linux/slab.h>
29#include <linux/export.h>
30#include <linux/if_vlan.h>
31#include <net/tcp.h>
32#include <net/udp.h>
33#include <net/addrconf.h>
34#include <net/ndisc.h>
35#include <net/ip6_checksum.h>
36#include <asm/unaligned.h>
37#include <trace/events/napi.h>
38
39/*
40 * We maintain a small pool of fully-sized skbs, to make sure the
41 * message gets out even in extreme OOM situations.
42 */
43
44#define MAX_UDP_CHUNK 1460
45#define MAX_SKBS 32
46
47static struct sk_buff_head skb_pool;
48
49DEFINE_STATIC_SRCU(netpoll_srcu);
50
51#define USEC_PER_POLL	50
52
53#define MAX_SKB_SIZE							\
54	(sizeof(struct ethhdr) +					\
55	 sizeof(struct iphdr) +						\
56	 sizeof(struct udphdr) +					\
57	 MAX_UDP_CHUNK)
58
59static void zap_completion_queue(void);
60static void netpoll_async_cleanup(struct work_struct *work);
61
62static unsigned int carrier_timeout = 4;
63module_param(carrier_timeout, uint, 0644);
64
65#define np_info(np, fmt, ...)				\
66	pr_info("%s: " fmt, np->name, ##__VA_ARGS__)
67#define np_err(np, fmt, ...)				\
68	pr_err("%s: " fmt, np->name, ##__VA_ARGS__)
69#define np_notice(np, fmt, ...)				\
70	pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)
71
72static int netpoll_start_xmit(struct sk_buff *skb, struct net_device *dev,
73			      struct netdev_queue *txq)
74{
75	int status = NETDEV_TX_OK;
76	netdev_features_t features;
77
78	features = netif_skb_features(skb);
79
80	if (skb_vlan_tag_present(skb) &&
81	    !vlan_hw_offload_capable(features, skb->vlan_proto)) {
82		skb = __vlan_hwaccel_push_inside(skb);
83		if (unlikely(!skb)) {
84			/* This is actually a packet drop, but we
85			 * don't want the code that calls this
86			 * function to try and operate on a NULL skb.
87			 */
88			goto out;
89		}
90	}
91
92	status = netdev_start_xmit(skb, dev, txq, false);
93
94out:
95	return status;
96}
97
98static void queue_process(struct work_struct *work)
99{
100	struct netpoll_info *npinfo =
101		container_of(work, struct netpoll_info, tx_work.work);
102	struct sk_buff *skb;
103	unsigned long flags;
104
105	while ((skb = skb_dequeue(&npinfo->txq))) {
106		struct net_device *dev = skb->dev;
107		struct netdev_queue *txq;
108
109		if (!netif_device_present(dev) || !netif_running(dev)) {
110			kfree_skb(skb);
111			continue;
112		}
113
114		txq = skb_get_tx_queue(dev, skb);
115
116		local_irq_save(flags);
117		HARD_TX_LOCK(dev, txq, smp_processor_id());
118		if (netif_xmit_frozen_or_stopped(txq) ||
119		    netpoll_start_xmit(skb, dev, txq) != NETDEV_TX_OK) {
120			skb_queue_head(&npinfo->txq, skb);
121			HARD_TX_UNLOCK(dev, txq);
122			local_irq_restore(flags);
123
124			schedule_delayed_work(&npinfo->tx_work, HZ/10);
125			return;
126		}
127		HARD_TX_UNLOCK(dev, txq);
128		local_irq_restore(flags);
129	}
130}
131
132/*
133 * Check whether delayed processing was scheduled for our NIC. If so,
134 * we attempt to grab the poll lock and use ->poll() to pump the card.
135 * If this fails, either we've recursed in ->poll() or it's already
136 * running on another CPU.
137 *
138 * Note: we don't mask interrupts with this lock because we're using
139 * trylock here and interrupts are already disabled in the softirq
140 * case. Further, we test the poll_owner to avoid recursion on UP
141 * systems where the lock doesn't exist.
142 */
143static int poll_one_napi(struct napi_struct *napi, int budget)
144{
145	int work;
146
147	/* net_rx_action's ->poll() invocations and our's are
148	 * synchronized by this test which is only made while
149	 * holding the napi->poll_lock.
150	 */
151	if (!test_bit(NAPI_STATE_SCHED, &napi->state))
152		return budget;
153
154	set_bit(NAPI_STATE_NPSVC, &napi->state);
155
156	work = napi->poll(napi, budget);
157	WARN_ONCE(work > budget, "%pF exceeded budget in poll\n", napi->poll);
158	trace_napi_poll(napi);
159
160	clear_bit(NAPI_STATE_NPSVC, &napi->state);
161
162	return budget - work;
163}
164
165static void poll_napi(struct net_device *dev, int budget)
166{
167	struct napi_struct *napi;
168
169	list_for_each_entry(napi, &dev->napi_list, dev_list) {
170		if (napi->poll_owner != smp_processor_id() &&
171		    spin_trylock(&napi->poll_lock)) {
172			budget = poll_one_napi(napi, budget);
173			spin_unlock(&napi->poll_lock);
174		}
175	}
176}
177
178static void netpoll_poll_dev(struct net_device *dev)
179{
180	const struct net_device_ops *ops;
181	struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
182	int budget = 0;
183
184	/* Don't do any rx activity if the dev_lock mutex is held
185	 * the dev_open/close paths use this to block netpoll activity
186	 * while changing device state
187	 */
188	if (down_trylock(&ni->dev_lock))
189		return;
190
191	if (!netif_running(dev)) {
192		up(&ni->dev_lock);
193		return;
194	}
195
196	ops = dev->netdev_ops;
197	if (!ops->ndo_poll_controller) {
198		up(&ni->dev_lock);
199		return;
200	}
201
202	/* Process pending work on NIC */
203	ops->ndo_poll_controller(dev);
204
205	poll_napi(dev, budget);
206
207	up(&ni->dev_lock);
208
209	zap_completion_queue();
210}
211
212void netpoll_poll_disable(struct net_device *dev)
213{
214	struct netpoll_info *ni;
215	int idx;
216	might_sleep();
217	idx = srcu_read_lock(&netpoll_srcu);
218	ni = srcu_dereference(dev->npinfo, &netpoll_srcu);
219	if (ni)
220		down(&ni->dev_lock);
221	srcu_read_unlock(&netpoll_srcu, idx);
222}
223EXPORT_SYMBOL(netpoll_poll_disable);
224
225void netpoll_poll_enable(struct net_device *dev)
226{
227	struct netpoll_info *ni;
228	rcu_read_lock();
229	ni = rcu_dereference(dev->npinfo);
230	if (ni)
231		up(&ni->dev_lock);
232	rcu_read_unlock();
233}
234EXPORT_SYMBOL(netpoll_poll_enable);
235
236static void refill_skbs(void)
237{
238	struct sk_buff *skb;
239	unsigned long flags;
240
241	spin_lock_irqsave(&skb_pool.lock, flags);
242	while (skb_pool.qlen < MAX_SKBS) {
243		skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
244		if (!skb)
245			break;
246
247		__skb_queue_tail(&skb_pool, skb);
248	}
249	spin_unlock_irqrestore(&skb_pool.lock, flags);
250}
251
252static void zap_completion_queue(void)
253{
254	unsigned long flags;
255	struct softnet_data *sd = &get_cpu_var(softnet_data);
256
257	if (sd->completion_queue) {
258		struct sk_buff *clist;
259
260		local_irq_save(flags);
261		clist = sd->completion_queue;
262		sd->completion_queue = NULL;
263		local_irq_restore(flags);
264
265		while (clist != NULL) {
266			struct sk_buff *skb = clist;
267			clist = clist->next;
268			if (!skb_irq_freeable(skb)) {
269				atomic_inc(&skb->users);
270				dev_kfree_skb_any(skb); /* put this one back */
271			} else {
272				__kfree_skb(skb);
273			}
274		}
275	}
276
277	put_cpu_var(softnet_data);
278}
279
280static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
281{
282	int count = 0;
283	struct sk_buff *skb;
284
285	zap_completion_queue();
286	refill_skbs();
287repeat:
288
289	skb = alloc_skb(len, GFP_ATOMIC);
290	if (!skb)
291		skb = skb_dequeue(&skb_pool);
292
293	if (!skb) {
294		if (++count < 10) {
295			netpoll_poll_dev(np->dev);
296			goto repeat;
297		}
298		return NULL;
299	}
300
301	atomic_set(&skb->users, 1);
302	skb_reserve(skb, reserve);
303	return skb;
304}
305
306static int netpoll_owner_active(struct net_device *dev)
307{
308	struct napi_struct *napi;
309
310	list_for_each_entry(napi, &dev->napi_list, dev_list) {
311		if (napi->poll_owner == smp_processor_id())
312			return 1;
313	}
314	return 0;
315}
316
317/* call with IRQ disabled */
318void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
319			     struct net_device *dev)
320{
321	int status = NETDEV_TX_BUSY;
322	unsigned long tries;
323	/* It is up to the caller to keep npinfo alive. */
324	struct netpoll_info *npinfo;
325
326	WARN_ON_ONCE(!irqs_disabled());
327
328	npinfo = rcu_dereference_bh(np->dev->npinfo);
329	if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
330		dev_kfree_skb_irq(skb);
331		return;
332	}
333
334	/* don't get messages out of order, and no recursion */
335	if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
336		struct netdev_queue *txq;
337
338		txq = netdev_pick_tx(dev, skb, NULL);
339
340		/* try until next clock tick */
341		for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
342		     tries > 0; --tries) {
343			if (HARD_TX_TRYLOCK(dev, txq)) {
344				if (!netif_xmit_stopped(txq))
345					status = netpoll_start_xmit(skb, dev, txq);
346
347				HARD_TX_UNLOCK(dev, txq);
348
349				if (status == NETDEV_TX_OK)
350					break;
351
352			}
353
354			/* tickle device maybe there is some cleanup */
355			netpoll_poll_dev(np->dev);
356
357			udelay(USEC_PER_POLL);
358		}
359
360		WARN_ONCE(!irqs_disabled(),
361			"netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pF)\n",
362			dev->name, dev->netdev_ops->ndo_start_xmit);
363
364	}
365
366	if (status != NETDEV_TX_OK) {
367		skb_queue_tail(&npinfo->txq, skb);
368		schedule_delayed_work(&npinfo->tx_work,0);
369	}
370}
371EXPORT_SYMBOL(netpoll_send_skb_on_dev);
372
373void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
374{
375	int total_len, ip_len, udp_len;
376	struct sk_buff *skb;
377	struct udphdr *udph;
378	struct iphdr *iph;
379	struct ethhdr *eth;
380	static atomic_t ip_ident;
381	struct ipv6hdr *ip6h;
382
383	udp_len = len + sizeof(*udph);
384	if (np->ipv6)
385		ip_len = udp_len + sizeof(*ip6h);
386	else
387		ip_len = udp_len + sizeof(*iph);
388
389	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
390
391	skb = find_skb(np, total_len + np->dev->needed_tailroom,
392		       total_len - len);
393	if (!skb)
394		return;
395
396	skb_copy_to_linear_data(skb, msg, len);
397	skb_put(skb, len);
398
399	skb_push(skb, sizeof(*udph));
400	skb_reset_transport_header(skb);
401	udph = udp_hdr(skb);
402	udph->source = htons(np->local_port);
403	udph->dest = htons(np->remote_port);
404	udph->len = htons(udp_len);
405
406	if (np->ipv6) {
407		udph->check = 0;
408		udph->check = csum_ipv6_magic(&np->local_ip.in6,
409					      &np->remote_ip.in6,
410					      udp_len, IPPROTO_UDP,
411					      csum_partial(udph, udp_len, 0));
412		if (udph->check == 0)
413			udph->check = CSUM_MANGLED_0;
414
415		skb_push(skb, sizeof(*ip6h));
416		skb_reset_network_header(skb);
417		ip6h = ipv6_hdr(skb);
418
419		/* ip6h->version = 6; ip6h->priority = 0; */
420		put_unaligned(0x60, (unsigned char *)ip6h);
421		ip6h->flow_lbl[0] = 0;
422		ip6h->flow_lbl[1] = 0;
423		ip6h->flow_lbl[2] = 0;
424
425		ip6h->payload_len = htons(sizeof(struct udphdr) + len);
426		ip6h->nexthdr = IPPROTO_UDP;
427		ip6h->hop_limit = 32;
428		ip6h->saddr = np->local_ip.in6;
429		ip6h->daddr = np->remote_ip.in6;
430
431		eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
432		skb_reset_mac_header(skb);
433		skb->protocol = eth->h_proto = htons(ETH_P_IPV6);
434	} else {
435		udph->check = 0;
436		udph->check = csum_tcpudp_magic(np->local_ip.ip,
437						np->remote_ip.ip,
438						udp_len, IPPROTO_UDP,
439						csum_partial(udph, udp_len, 0));
440		if (udph->check == 0)
441			udph->check = CSUM_MANGLED_0;
442
443		skb_push(skb, sizeof(*iph));
444		skb_reset_network_header(skb);
445		iph = ip_hdr(skb);
446
447		/* iph->version = 4; iph->ihl = 5; */
448		put_unaligned(0x45, (unsigned char *)iph);
449		iph->tos      = 0;
450		put_unaligned(htons(ip_len), &(iph->tot_len));
451		iph->id       = htons(atomic_inc_return(&ip_ident));
452		iph->frag_off = 0;
453		iph->ttl      = 64;
454		iph->protocol = IPPROTO_UDP;
455		iph->check    = 0;
456		put_unaligned(np->local_ip.ip, &(iph->saddr));
457		put_unaligned(np->remote_ip.ip, &(iph->daddr));
458		iph->check    = ip_fast_csum((unsigned char *)iph, iph->ihl);
459
460		eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
461		skb_reset_mac_header(skb);
462		skb->protocol = eth->h_proto = htons(ETH_P_IP);
463	}
464
465	ether_addr_copy(eth->h_source, np->dev->dev_addr);
466	ether_addr_copy(eth->h_dest, np->remote_mac);
467
468	skb->dev = np->dev;
469
470	netpoll_send_skb(np, skb);
471}
472EXPORT_SYMBOL(netpoll_send_udp);
473
474void netpoll_print_options(struct netpoll *np)
475{
476	np_info(np, "local port %d\n", np->local_port);
477	if (np->ipv6)
478		np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
479	else
480		np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
481	np_info(np, "interface '%s'\n", np->dev_name);
482	np_info(np, "remote port %d\n", np->remote_port);
483	if (np->ipv6)
484		np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
485	else
486		np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
487	np_info(np, "remote ethernet address %pM\n", np->remote_mac);
488}
489EXPORT_SYMBOL(netpoll_print_options);
490
491static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
492{
493	const char *end;
494
495	if (!strchr(str, ':') &&
496	    in4_pton(str, -1, (void *)addr, -1, &end) > 0) {
497		if (!*end)
498			return 0;
499	}
500	if (in6_pton(str, -1, addr->in6.s6_addr, -1, &end) > 0) {
501#if IS_ENABLED(CONFIG_IPV6)
502		if (!*end)
503			return 1;
504#else
505		return -1;
506#endif
507	}
508	return -1;
509}
510
511int netpoll_parse_options(struct netpoll *np, char *opt)
512{
513	char *cur=opt, *delim;
514	int ipv6;
515	bool ipversion_set = false;
516
517	if (*cur != '@') {
518		if ((delim = strchr(cur, '@')) == NULL)
519			goto parse_failed;
520		*delim = 0;
521		if (kstrtou16(cur, 10, &np->local_port))
522			goto parse_failed;
523		cur = delim;
524	}
525	cur++;
526
527	if (*cur != '/') {
528		ipversion_set = true;
529		if ((delim = strchr(cur, '/')) == NULL)
530			goto parse_failed;
531		*delim = 0;
532		ipv6 = netpoll_parse_ip_addr(cur, &np->local_ip);
533		if (ipv6 < 0)
534			goto parse_failed;
535		else
536			np->ipv6 = (bool)ipv6;
537		cur = delim;
538	}
539	cur++;
540
541	if (*cur != ',') {
542		/* parse out dev name */
543		if ((delim = strchr(cur, ',')) == NULL)
544			goto parse_failed;
545		*delim = 0;
546		strlcpy(np->dev_name, cur, sizeof(np->dev_name));
547		cur = delim;
548	}
549	cur++;
550
551	if (*cur != '@') {
552		/* dst port */
553		if ((delim = strchr(cur, '@')) == NULL)
554			goto parse_failed;
555		*delim = 0;
556		if (*cur == ' ' || *cur == '\t')
557			np_info(np, "warning: whitespace is not allowed\n");
558		if (kstrtou16(cur, 10, &np->remote_port))
559			goto parse_failed;
560		cur = delim;
561	}
562	cur++;
563
564	/* dst ip */
565	if ((delim = strchr(cur, '/')) == NULL)
566		goto parse_failed;
567	*delim = 0;
568	ipv6 = netpoll_parse_ip_addr(cur, &np->remote_ip);
569	if (ipv6 < 0)
570		goto parse_failed;
571	else if (ipversion_set && np->ipv6 != (bool)ipv6)
572		goto parse_failed;
573	else
574		np->ipv6 = (bool)ipv6;
575	cur = delim + 1;
576
577	if (*cur != 0) {
578		/* MAC address */
579		if (!mac_pton(cur, np->remote_mac))
580			goto parse_failed;
581	}
582
583	netpoll_print_options(np);
584
585	return 0;
586
587 parse_failed:
588	np_info(np, "couldn't parse config at '%s'!\n", cur);
589	return -1;
590}
591EXPORT_SYMBOL(netpoll_parse_options);
592
593int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
594{
595	struct netpoll_info *npinfo;
596	const struct net_device_ops *ops;
597	int err;
598
599	np->dev = ndev;
600	strlcpy(np->dev_name, ndev->name, IFNAMSIZ);
601	INIT_WORK(&np->cleanup_work, netpoll_async_cleanup);
602
603	if ((ndev->priv_flags & IFF_DISABLE_NETPOLL) ||
604	    !ndev->netdev_ops->ndo_poll_controller) {
605		np_err(np, "%s doesn't support polling, aborting\n",
606		       np->dev_name);
607		err = -ENOTSUPP;
608		goto out;
609	}
610
611	if (!ndev->npinfo) {
612		npinfo = kmalloc(sizeof(*npinfo), GFP_KERNEL);
613		if (!npinfo) {
614			err = -ENOMEM;
615			goto out;
616		}
617
618		sema_init(&npinfo->dev_lock, 1);
619		skb_queue_head_init(&npinfo->txq);
620		INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);
621
622		atomic_set(&npinfo->refcnt, 1);
623
624		ops = np->dev->netdev_ops;
625		if (ops->ndo_netpoll_setup) {
626			err = ops->ndo_netpoll_setup(ndev, npinfo);
627			if (err)
628				goto free_npinfo;
629		}
630	} else {
631		npinfo = rtnl_dereference(ndev->npinfo);
632		atomic_inc(&npinfo->refcnt);
633	}
634
635	npinfo->netpoll = np;
636
637	/* last thing to do is link it to the net device structure */
638	rcu_assign_pointer(ndev->npinfo, npinfo);
639
640	return 0;
641
642free_npinfo:
643	kfree(npinfo);
644out:
645	return err;
646}
647EXPORT_SYMBOL_GPL(__netpoll_setup);
648
649int netpoll_setup(struct netpoll *np)
650{
651	struct net_device *ndev = NULL;
652	struct in_device *in_dev;
653	int err;
654
655	rtnl_lock();
656	if (np->dev_name) {
657		struct net *net = current->nsproxy->net_ns;
658		ndev = __dev_get_by_name(net, np->dev_name);
659	}
660	if (!ndev) {
661		np_err(np, "%s doesn't exist, aborting\n", np->dev_name);
662		err = -ENODEV;
663		goto unlock;
664	}
665	dev_hold(ndev);
666
667	if (netdev_master_upper_dev_get(ndev)) {
668		np_err(np, "%s is a slave device, aborting\n", np->dev_name);
669		err = -EBUSY;
670		goto put;
671	}
672
673	if (!netif_running(ndev)) {
674		unsigned long atmost, atleast;
675
676		np_info(np, "device %s not up yet, forcing it\n", np->dev_name);
677
678		err = dev_open(ndev);
679
680		if (err) {
681			np_err(np, "failed to open %s\n", ndev->name);
682			goto put;
683		}
684
685		rtnl_unlock();
686		atleast = jiffies + HZ/10;
687		atmost = jiffies + carrier_timeout * HZ;
688		while (!netif_carrier_ok(ndev)) {
689			if (time_after(jiffies, atmost)) {
690				np_notice(np, "timeout waiting for carrier\n");
691				break;
692			}
693			msleep(1);
694		}
695
696		/* If carrier appears to come up instantly, we don't
697		 * trust it and pause so that we don't pump all our
698		 * queued console messages into the bitbucket.
699		 */
700
701		if (time_before(jiffies, atleast)) {
702			np_notice(np, "carrier detect appears untrustworthy, waiting 4 seconds\n");
703			msleep(4000);
704		}
705		rtnl_lock();
706	}
707
708	if (!np->local_ip.ip) {
709		if (!np->ipv6) {
710			in_dev = __in_dev_get_rtnl(ndev);
711
712			if (!in_dev || !in_dev->ifa_list) {
713				np_err(np, "no IP address for %s, aborting\n",
714				       np->dev_name);
715				err = -EDESTADDRREQ;
716				goto put;
717			}
718
719			np->local_ip.ip = in_dev->ifa_list->ifa_local;
720			np_info(np, "local IP %pI4\n", &np->local_ip.ip);
721		} else {
722#if IS_ENABLED(CONFIG_IPV6)
723			struct inet6_dev *idev;
724
725			err = -EDESTADDRREQ;
726			idev = __in6_dev_get(ndev);
727			if (idev) {
728				struct inet6_ifaddr *ifp;
729
730				read_lock_bh(&idev->lock);
731				list_for_each_entry(ifp, &idev->addr_list, if_list) {
732					if (ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL)
733						continue;
734					np->local_ip.in6 = ifp->addr;
735					err = 0;
736					break;
737				}
738				read_unlock_bh(&idev->lock);
739			}
740			if (err) {
741				np_err(np, "no IPv6 address for %s, aborting\n",
742				       np->dev_name);
743				goto put;
744			} else
745				np_info(np, "local IPv6 %pI6c\n", &np->local_ip.in6);
746#else
747			np_err(np, "IPv6 is not supported %s, aborting\n",
748			       np->dev_name);
749			err = -EINVAL;
750			goto put;
751#endif
752		}
753	}
754
755	/* fill up the skb queue */
756	refill_skbs();
757
758	err = __netpoll_setup(np, ndev);
759	if (err)
760		goto put;
761
762	rtnl_unlock();
763	return 0;
764
765put:
766	dev_put(ndev);
767unlock:
768	rtnl_unlock();
769	return err;
770}
771EXPORT_SYMBOL(netpoll_setup);
772
773static int __init netpoll_init(void)
774{
775	skb_queue_head_init(&skb_pool);
776	return 0;
777}
778core_initcall(netpoll_init);
779
780static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
781{
782	struct netpoll_info *npinfo =
783			container_of(rcu_head, struct netpoll_info, rcu);
784
785	skb_queue_purge(&npinfo->txq);
786
787	/* we can't call cancel_delayed_work_sync here, as we are in softirq */
788	cancel_delayed_work(&npinfo->tx_work);
789
790	/* clean after last, unfinished work */
791	__skb_queue_purge(&npinfo->txq);
792	/* now cancel it again */
793	cancel_delayed_work(&npinfo->tx_work);
794	kfree(npinfo);
795}
796
797void __netpoll_cleanup(struct netpoll *np)
798{
799	struct netpoll_info *npinfo;
800
801	/* rtnl_dereference would be preferable here but
802	 * rcu_cleanup_netpoll path can put us in here safely without
803	 * holding the rtnl, so plain rcu_dereference it is
804	 */
805	npinfo = rtnl_dereference(np->dev->npinfo);
806	if (!npinfo)
807		return;
808
809	synchronize_srcu(&netpoll_srcu);
810
811	if (atomic_dec_and_test(&npinfo->refcnt)) {
812		const struct net_device_ops *ops;
813
814		ops = np->dev->netdev_ops;
815		if (ops->ndo_netpoll_cleanup)
816			ops->ndo_netpoll_cleanup(np->dev);
817
818		RCU_INIT_POINTER(np->dev->npinfo, NULL);
819		call_rcu_bh(&npinfo->rcu, rcu_cleanup_netpoll_info);
820	} else
821		RCU_INIT_POINTER(np->dev->npinfo, NULL);
822}
823EXPORT_SYMBOL_GPL(__netpoll_cleanup);
824
825static void netpoll_async_cleanup(struct work_struct *work)
826{
827	struct netpoll *np = container_of(work, struct netpoll, cleanup_work);
828
829	rtnl_lock();
830	__netpoll_cleanup(np);
831	rtnl_unlock();
832	kfree(np);
833}
834
835void __netpoll_free_async(struct netpoll *np)
836{
837	schedule_work(&np->cleanup_work);
838}
839EXPORT_SYMBOL_GPL(__netpoll_free_async);
840
841void netpoll_cleanup(struct netpoll *np)
842{
843	rtnl_lock();
844	if (!np->dev)
845		goto out;
846	__netpoll_cleanup(np);
847	dev_put(np->dev);
848	np->dev = NULL;
849out:
850	rtnl_unlock();
851}
852EXPORT_SYMBOL(netpoll_cleanup);
853