1#ifndef _ASM_X86_APIC_H
2#define _ASM_X86_APIC_H
3
4#include <linux/cpumask.h>
5#include <linux/pm.h>
6
7#include <asm/alternative.h>
8#include <asm/cpufeature.h>
9#include <asm/processor.h>
10#include <asm/apicdef.h>
11#include <linux/atomic.h>
12#include <asm/fixmap.h>
13#include <asm/mpspec.h>
14#include <asm/msr.h>
15#include <asm/idle.h>
16
17#define ARCH_APICTIMER_STOPS_ON_C3	1
18
19/*
20 * Debugging macros
21 */
22#define APIC_QUIET   0
23#define APIC_VERBOSE 1
24#define APIC_DEBUG   2
25
26/*
27 * Define the default level of output to be very little
28 * This can be turned up by using apic=verbose for more
29 * information and apic=debug for _lots_ of information.
30 * apic_verbosity is defined in apic.c
31 */
32#define apic_printk(v, s, a...) do {       \
33		if ((v) <= apic_verbosity) \
34			printk(s, ##a);    \
35	} while (0)
36
37
38#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
39extern void generic_apic_probe(void);
40#else
41static inline void generic_apic_probe(void)
42{
43}
44#endif
45
46#ifdef CONFIG_X86_LOCAL_APIC
47
48extern unsigned int apic_verbosity;
49extern int local_apic_timer_c2_ok;
50
51extern int disable_apic;
52extern unsigned int lapic_timer_frequency;
53
54#ifdef CONFIG_SMP
55extern void __inquire_remote_apic(int apicid);
56#else /* CONFIG_SMP */
57static inline void __inquire_remote_apic(int apicid)
58{
59}
60#endif /* CONFIG_SMP */
61
62static inline void default_inquire_remote_apic(int apicid)
63{
64	if (apic_verbosity >= APIC_DEBUG)
65		__inquire_remote_apic(apicid);
66}
67
68/*
69 * With 82489DX we can't rely on apic feature bit
70 * retrieved via cpuid but still have to deal with
71 * such an apic chip so we assume that SMP configuration
72 * is found from MP table (64bit case uses ACPI mostly
73 * which set smp presence flag as well so we are safe
74 * to use this helper too).
75 */
76static inline bool apic_from_smp_config(void)
77{
78	return smp_found_config && !disable_apic;
79}
80
81/*
82 * Basic functions accessing APICs.
83 */
84#ifdef CONFIG_PARAVIRT
85#include <asm/paravirt.h>
86#endif
87
88extern int setup_profiling_timer(unsigned int);
89
90static inline void native_apic_mem_write(u32 reg, u32 v)
91{
92	volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg);
93
94	alternative_io("movl %0, %P1", "xchgl %0, %P1", X86_BUG_11AP,
95		       ASM_OUTPUT2("=r" (v), "=m" (*addr)),
96		       ASM_OUTPUT2("0" (v), "m" (*addr)));
97}
98
99static inline u32 native_apic_mem_read(u32 reg)
100{
101	return *((volatile u32 *)(APIC_BASE + reg));
102}
103
104extern void native_apic_wait_icr_idle(void);
105extern u32 native_safe_apic_wait_icr_idle(void);
106extern void native_apic_icr_write(u32 low, u32 id);
107extern u64 native_apic_icr_read(void);
108
109static inline bool apic_is_x2apic_enabled(void)
110{
111	u64 msr;
112
113	if (rdmsrl_safe(MSR_IA32_APICBASE, &msr))
114		return false;
115	return msr & X2APIC_ENABLE;
116}
117
118#ifdef CONFIG_X86_X2APIC
119/*
120 * Make previous memory operations globally visible before
121 * sending the IPI through x2apic wrmsr. We need a serializing instruction or
122 * mfence for this.
123 */
124static inline void x2apic_wrmsr_fence(void)
125{
126	asm volatile("mfence" : : : "memory");
127}
128
129static inline void native_apic_msr_write(u32 reg, u32 v)
130{
131	if (reg == APIC_DFR || reg == APIC_ID || reg == APIC_LDR ||
132	    reg == APIC_LVR)
133		return;
134
135	wrmsr(APIC_BASE_MSR + (reg >> 4), v, 0);
136}
137
138static inline void native_apic_msr_eoi_write(u32 reg, u32 v)
139{
140	wrmsr(APIC_BASE_MSR + (APIC_EOI >> 4), APIC_EOI_ACK, 0);
141}
142
143static inline u32 native_apic_msr_read(u32 reg)
144{
145	u64 msr;
146
147	if (reg == APIC_DFR)
148		return -1;
149
150	rdmsrl(APIC_BASE_MSR + (reg >> 4), msr);
151	return (u32)msr;
152}
153
154static inline void native_x2apic_wait_icr_idle(void)
155{
156	/* no need to wait for icr idle in x2apic */
157	return;
158}
159
160static inline u32 native_safe_x2apic_wait_icr_idle(void)
161{
162	/* no need to wait for icr idle in x2apic */
163	return 0;
164}
165
166static inline void native_x2apic_icr_write(u32 low, u32 id)
167{
168	wrmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), ((__u64) id) << 32 | low);
169}
170
171static inline u64 native_x2apic_icr_read(void)
172{
173	unsigned long val;
174
175	rdmsrl(APIC_BASE_MSR + (APIC_ICR >> 4), val);
176	return val;
177}
178
179extern int x2apic_mode;
180extern int x2apic_phys;
181extern void __init check_x2apic(void);
182extern void x2apic_setup(void);
183static inline int x2apic_enabled(void)
184{
185	return cpu_has_x2apic && apic_is_x2apic_enabled();
186}
187
188#define x2apic_supported()	(cpu_has_x2apic)
189#else
190static inline void check_x2apic(void) { }
191static inline void x2apic_setup(void) { }
192static inline int x2apic_enabled(void) { return 0; }
193
194#define x2apic_mode		(0)
195#define	x2apic_supported()	(0)
196#endif
197
198extern void enable_IR_x2apic(void);
199
200extern int get_physical_broadcast(void);
201
202extern int lapic_get_maxlvt(void);
203extern void clear_local_APIC(void);
204extern void disconnect_bsp_APIC(int virt_wire_setup);
205extern void disable_local_APIC(void);
206extern void lapic_shutdown(void);
207extern void sync_Arb_IDs(void);
208extern void init_bsp_APIC(void);
209extern void setup_local_APIC(void);
210extern void init_apic_mappings(void);
211void register_lapic_address(unsigned long address);
212extern void setup_boot_APIC_clock(void);
213extern void setup_secondary_APIC_clock(void);
214extern int APIC_init_uniprocessor(void);
215
216#ifdef CONFIG_X86_64
217static inline int apic_force_enable(unsigned long addr)
218{
219	return -1;
220}
221#else
222extern int apic_force_enable(unsigned long addr);
223#endif
224
225extern int apic_bsp_setup(bool upmode);
226extern void apic_ap_setup(void);
227
228/*
229 * On 32bit this is mach-xxx local
230 */
231#ifdef CONFIG_X86_64
232extern int apic_is_clustered_box(void);
233#else
234static inline int apic_is_clustered_box(void)
235{
236	return 0;
237}
238#endif
239
240extern int setup_APIC_eilvt(u8 lvt_off, u8 vector, u8 msg_type, u8 mask);
241
242#else /* !CONFIG_X86_LOCAL_APIC */
243static inline void lapic_shutdown(void) { }
244#define local_apic_timer_c2_ok		1
245static inline void init_apic_mappings(void) { }
246static inline void disable_local_APIC(void) { }
247# define setup_boot_APIC_clock x86_init_noop
248# define setup_secondary_APIC_clock x86_init_noop
249#endif /* !CONFIG_X86_LOCAL_APIC */
250
251#ifdef CONFIG_X86_64
252#define	SET_APIC_ID(x)		(apic->set_apic_id(x))
253#else
254
255#endif
256
257/*
258 * Copyright 2004 James Cleverdon, IBM.
259 * Subject to the GNU Public License, v.2
260 *
261 * Generic APIC sub-arch data struct.
262 *
263 * Hacked for x86-64 by James Cleverdon from i386 architecture code by
264 * Martin Bligh, Andi Kleen, James Bottomley, John Stultz, and
265 * James Cleverdon.
266 */
267struct apic {
268	char *name;
269
270	int (*probe)(void);
271	int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id);
272	int (*apic_id_valid)(int apicid);
273	int (*apic_id_registered)(void);
274
275	u32 irq_delivery_mode;
276	u32 irq_dest_mode;
277
278	const struct cpumask *(*target_cpus)(void);
279
280	int disable_esr;
281
282	int dest_logical;
283	unsigned long (*check_apicid_used)(physid_mask_t *map, int apicid);
284
285	void (*vector_allocation_domain)(int cpu, struct cpumask *retmask,
286					 const struct cpumask *mask);
287	void (*init_apic_ldr)(void);
288
289	void (*ioapic_phys_id_map)(physid_mask_t *phys_map, physid_mask_t *retmap);
290
291	void (*setup_apic_routing)(void);
292	int (*cpu_present_to_apicid)(int mps_cpu);
293	void (*apicid_to_cpu_present)(int phys_apicid, physid_mask_t *retmap);
294	int (*check_phys_apicid_present)(int phys_apicid);
295	int (*phys_pkg_id)(int cpuid_apic, int index_msb);
296
297	unsigned int (*get_apic_id)(unsigned long x);
298	unsigned long (*set_apic_id)(unsigned int id);
299	unsigned long apic_id_mask;
300
301	int (*cpu_mask_to_apicid_and)(const struct cpumask *cpumask,
302				      const struct cpumask *andmask,
303				      unsigned int *apicid);
304
305	/* ipi */
306	void (*send_IPI_mask)(const struct cpumask *mask, int vector);
307	void (*send_IPI_mask_allbutself)(const struct cpumask *mask,
308					 int vector);
309	void (*send_IPI_allbutself)(int vector);
310	void (*send_IPI_all)(int vector);
311	void (*send_IPI_self)(int vector);
312
313	/* wakeup_secondary_cpu */
314	int (*wakeup_secondary_cpu)(int apicid, unsigned long start_eip);
315
316	bool wait_for_init_deassert;
317	void (*inquire_remote_apic)(int apicid);
318
319	/* apic ops */
320	u32 (*read)(u32 reg);
321	void (*write)(u32 reg, u32 v);
322	/*
323	 * ->eoi_write() has the same signature as ->write().
324	 *
325	 * Drivers can support both ->eoi_write() and ->write() by passing the same
326	 * callback value. Kernel can override ->eoi_write() and fall back
327	 * on write for EOI.
328	 */
329	void (*eoi_write)(u32 reg, u32 v);
330	u64 (*icr_read)(void);
331	void (*icr_write)(u32 low, u32 high);
332	void (*wait_icr_idle)(void);
333	u32 (*safe_wait_icr_idle)(void);
334
335#ifdef CONFIG_X86_32
336	/*
337	 * Called very early during boot from get_smp_config().  It should
338	 * return the logical apicid.  x86_[bios]_cpu_to_apicid is
339	 * initialized before this function is called.
340	 *
341	 * If logical apicid can't be determined that early, the function
342	 * may return BAD_APICID.  Logical apicid will be configured after
343	 * init_apic_ldr() while bringing up CPUs.  Note that NUMA affinity
344	 * won't be applied properly during early boot in this case.
345	 */
346	int (*x86_32_early_logical_apicid)(int cpu);
347#endif
348};
349
350/*
351 * Pointer to the local APIC driver in use on this system (there's
352 * always just one such driver in use - the kernel decides via an
353 * early probing process which one it picks - and then sticks to it):
354 */
355extern struct apic *apic;
356
357/*
358 * APIC drivers are probed based on how they are listed in the .apicdrivers
359 * section. So the order is important and enforced by the ordering
360 * of different apic driver files in the Makefile.
361 *
362 * For the files having two apic drivers, we use apic_drivers()
363 * to enforce the order with in them.
364 */
365#define apic_driver(sym)					\
366	static const struct apic *__apicdrivers_##sym __used		\
367	__aligned(sizeof(struct apic *))			\
368	__section(.apicdrivers) = { &sym }
369
370#define apic_drivers(sym1, sym2)					\
371	static struct apic *__apicdrivers_##sym1##sym2[2] __used	\
372	__aligned(sizeof(struct apic *))				\
373	__section(.apicdrivers) = { &sym1, &sym2 }
374
375extern struct apic *__apicdrivers[], *__apicdrivers_end[];
376
377/*
378 * APIC functionality to boot other CPUs - only used on SMP:
379 */
380#ifdef CONFIG_SMP
381extern atomic_t init_deasserted;
382extern int wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip);
383#endif
384
385#ifdef CONFIG_X86_LOCAL_APIC
386
387static inline u32 apic_read(u32 reg)
388{
389	return apic->read(reg);
390}
391
392static inline void apic_write(u32 reg, u32 val)
393{
394	apic->write(reg, val);
395}
396
397static inline void apic_eoi(void)
398{
399	apic->eoi_write(APIC_EOI, APIC_EOI_ACK);
400}
401
402static inline u64 apic_icr_read(void)
403{
404	return apic->icr_read();
405}
406
407static inline void apic_icr_write(u32 low, u32 high)
408{
409	apic->icr_write(low, high);
410}
411
412static inline void apic_wait_icr_idle(void)
413{
414	apic->wait_icr_idle();
415}
416
417static inline u32 safe_apic_wait_icr_idle(void)
418{
419	return apic->safe_wait_icr_idle();
420}
421
422extern void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v));
423
424#else /* CONFIG_X86_LOCAL_APIC */
425
426static inline u32 apic_read(u32 reg) { return 0; }
427static inline void apic_write(u32 reg, u32 val) { }
428static inline void apic_eoi(void) { }
429static inline u64 apic_icr_read(void) { return 0; }
430static inline void apic_icr_write(u32 low, u32 high) { }
431static inline void apic_wait_icr_idle(void) { }
432static inline u32 safe_apic_wait_icr_idle(void) { return 0; }
433static inline void apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) {}
434
435#endif /* CONFIG_X86_LOCAL_APIC */
436
437static inline void ack_APIC_irq(void)
438{
439	/*
440	 * ack_APIC_irq() actually gets compiled as a single instruction
441	 * ... yummie.
442	 */
443	apic_eoi();
444}
445
446static inline unsigned default_get_apic_id(unsigned long x)
447{
448	unsigned int ver = GET_APIC_VERSION(apic_read(APIC_LVR));
449
450	if (APIC_XAPIC(ver) || boot_cpu_has(X86_FEATURE_EXTD_APICID))
451		return (x >> 24) & 0xFF;
452	else
453		return (x >> 24) & 0x0F;
454}
455
456/*
457 * Warm reset vector position:
458 */
459#define TRAMPOLINE_PHYS_LOW		0x467
460#define TRAMPOLINE_PHYS_HIGH		0x469
461
462#ifdef CONFIG_X86_64
463extern void apic_send_IPI_self(int vector);
464
465DECLARE_PER_CPU(int, x2apic_extra_bits);
466
467extern int default_cpu_present_to_apicid(int mps_cpu);
468extern int default_check_phys_apicid_present(int phys_apicid);
469#endif
470
471extern void generic_bigsmp_probe(void);
472
473
474#ifdef CONFIG_X86_LOCAL_APIC
475
476#include <asm/smp.h>
477
478#define APIC_DFR_VALUE	(APIC_DFR_FLAT)
479
480static inline const struct cpumask *default_target_cpus(void)
481{
482#ifdef CONFIG_SMP
483	return cpu_online_mask;
484#else
485	return cpumask_of(0);
486#endif
487}
488
489static inline const struct cpumask *online_target_cpus(void)
490{
491	return cpu_online_mask;
492}
493
494DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
495
496
497static inline unsigned int read_apic_id(void)
498{
499	unsigned int reg;
500
501	reg = apic_read(APIC_ID);
502
503	return apic->get_apic_id(reg);
504}
505
506static inline int default_apic_id_valid(int apicid)
507{
508	return (apicid < 255);
509}
510
511extern int default_acpi_madt_oem_check(char *, char *);
512
513extern void default_setup_apic_routing(void);
514
515extern struct apic apic_noop;
516
517#ifdef CONFIG_X86_32
518
519static inline int noop_x86_32_early_logical_apicid(int cpu)
520{
521	return BAD_APICID;
522}
523
524/*
525 * Set up the logical destination ID.
526 *
527 * Intel recommends to set DFR, LDR and TPR before enabling
528 * an APIC.  See e.g. "AP-388 82489DX User's Manual" (Intel
529 * document number 292116).  So here it goes...
530 */
531extern void default_init_apic_ldr(void);
532
533static inline int default_apic_id_registered(void)
534{
535	return physid_isset(read_apic_id(), phys_cpu_present_map);
536}
537
538static inline int default_phys_pkg_id(int cpuid_apic, int index_msb)
539{
540	return cpuid_apic >> index_msb;
541}
542
543#endif
544
545static inline int
546flat_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
547			    const struct cpumask *andmask,
548			    unsigned int *apicid)
549{
550	unsigned long cpu_mask = cpumask_bits(cpumask)[0] &
551				 cpumask_bits(andmask)[0] &
552				 cpumask_bits(cpu_online_mask)[0] &
553				 APIC_ALL_CPUS;
554
555	if (likely(cpu_mask)) {
556		*apicid = (unsigned int)cpu_mask;
557		return 0;
558	} else {
559		return -EINVAL;
560	}
561}
562
563extern int
564default_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
565			       const struct cpumask *andmask,
566			       unsigned int *apicid);
567
568static inline void
569flat_vector_allocation_domain(int cpu, struct cpumask *retmask,
570			      const struct cpumask *mask)
571{
572	/* Careful. Some cpus do not strictly honor the set of cpus
573	 * specified in the interrupt destination when using lowest
574	 * priority interrupt delivery mode.
575	 *
576	 * In particular there was a hyperthreading cpu observed to
577	 * deliver interrupts to the wrong hyperthread when only one
578	 * hyperthread was specified in the interrupt desitination.
579	 */
580	cpumask_clear(retmask);
581	cpumask_bits(retmask)[0] = APIC_ALL_CPUS;
582}
583
584static inline void
585default_vector_allocation_domain(int cpu, struct cpumask *retmask,
586				 const struct cpumask *mask)
587{
588	cpumask_copy(retmask, cpumask_of(cpu));
589}
590
591static inline unsigned long default_check_apicid_used(physid_mask_t *map, int apicid)
592{
593	return physid_isset(apicid, *map);
594}
595
596static inline void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap)
597{
598	*retmap = *phys_map;
599}
600
601static inline int __default_cpu_present_to_apicid(int mps_cpu)
602{
603	if (mps_cpu < nr_cpu_ids && cpu_present(mps_cpu))
604		return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu);
605	else
606		return BAD_APICID;
607}
608
609static inline int
610__default_check_phys_apicid_present(int phys_apicid)
611{
612	return physid_isset(phys_apicid, phys_cpu_present_map);
613}
614
615#ifdef CONFIG_X86_32
616static inline int default_cpu_present_to_apicid(int mps_cpu)
617{
618	return __default_cpu_present_to_apicid(mps_cpu);
619}
620
621static inline int
622default_check_phys_apicid_present(int phys_apicid)
623{
624	return __default_check_phys_apicid_present(phys_apicid);
625}
626#else
627extern int default_cpu_present_to_apicid(int mps_cpu);
628extern int default_check_phys_apicid_present(int phys_apicid);
629#endif
630
631#endif /* CONFIG_X86_LOCAL_APIC */
632extern void irq_enter(void);
633extern void irq_exit(void);
634
635static inline void entering_irq(void)
636{
637	irq_enter();
638	exit_idle();
639}
640
641static inline void entering_ack_irq(void)
642{
643	entering_irq();
644	ack_APIC_irq();
645}
646
647static inline void exiting_irq(void)
648{
649	irq_exit();
650}
651
652static inline void exiting_ack_irq(void)
653{
654	irq_exit();
655	/* Ack only at the end to avoid potential reentry */
656	ack_APIC_irq();
657}
658
659extern void ioapic_zap_locks(void);
660
661#endif /* _ASM_X86_APIC_H */
662