1#ifndef _ASM_X86_BARRIER_H
2#define _ASM_X86_BARRIER_H
3
4#include <asm/alternative.h>
5#include <asm/nops.h>
6
7/*
8 * Force strict CPU ordering.
9 * And yes, this is required on UP too when we're talking
10 * to devices.
11 */
12
13#ifdef CONFIG_X86_32
14/*
15 * Some non-Intel clones support out of order store. wmb() ceases to be a
16 * nop for these.
17 */
18#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
19#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
20#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
21#else
22#define mb() 	asm volatile("mfence":::"memory")
23#define rmb()	asm volatile("lfence":::"memory")
24#define wmb()	asm volatile("sfence" ::: "memory")
25#endif
26
27#ifdef CONFIG_X86_PPRO_FENCE
28#define dma_rmb()	rmb()
29#else
30#define dma_rmb()	barrier()
31#endif
32#define dma_wmb()	barrier()
33
34#ifdef CONFIG_SMP
35#define smp_mb()	mb()
36#define smp_rmb()	dma_rmb()
37#define smp_wmb()	barrier()
38#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
39#else /* !SMP */
40#define smp_mb()	barrier()
41#define smp_rmb()	barrier()
42#define smp_wmb()	barrier()
43#define set_mb(var, value) do { var = value; barrier(); } while (0)
44#endif /* SMP */
45
46#define read_barrier_depends()		do { } while (0)
47#define smp_read_barrier_depends()	do { } while (0)
48
49#if defined(CONFIG_X86_PPRO_FENCE)
50
51/*
52 * For this option x86 doesn't have a strong TSO memory
53 * model and we should fall back to full barriers.
54 */
55
56#define smp_store_release(p, v)						\
57do {									\
58	compiletime_assert_atomic_type(*p);				\
59	smp_mb();							\
60	ACCESS_ONCE(*p) = (v);						\
61} while (0)
62
63#define smp_load_acquire(p)						\
64({									\
65	typeof(*p) ___p1 = ACCESS_ONCE(*p);				\
66	compiletime_assert_atomic_type(*p);				\
67	smp_mb();							\
68	___p1;								\
69})
70
71#else /* regular x86 TSO memory ordering */
72
73#define smp_store_release(p, v)						\
74do {									\
75	compiletime_assert_atomic_type(*p);				\
76	barrier();							\
77	ACCESS_ONCE(*p) = (v);						\
78} while (0)
79
80#define smp_load_acquire(p)						\
81({									\
82	typeof(*p) ___p1 = ACCESS_ONCE(*p);				\
83	compiletime_assert_atomic_type(*p);				\
84	barrier();							\
85	___p1;								\
86})
87
88#endif
89
90/* Atomic operations are already serializing on x86 */
91#define smp_mb__before_atomic()	barrier()
92#define smp_mb__after_atomic()	barrier()
93
94/*
95 * Stop RDTSC speculation. This is needed when you need to use RDTSC
96 * (or get_cycles or vread that possibly accesses the TSC) in a defined
97 * code region.
98 */
99static __always_inline void rdtsc_barrier(void)
100{
101	alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC,
102			  "lfence", X86_FEATURE_LFENCE_RDTSC);
103}
104
105#endif /* _ASM_X86_BARRIER_H */
106