1/* 2 * Based on arch/arm/include/asm/cmpxchg.h 3 * 4 * Copyright (C) 2012 ARM Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18#ifndef __ASM_CMPXCHG_H 19#define __ASM_CMPXCHG_H 20 21#include <linux/bug.h> 22#include <linux/mmdebug.h> 23 24#include <asm/barrier.h> 25 26static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 27{ 28 unsigned long ret, tmp; 29 30 switch (size) { 31 case 1: 32 asm volatile("// __xchg1\n" 33 "1: ldxrb %w0, %2\n" 34 " stlxrb %w1, %w3, %2\n" 35 " cbnz %w1, 1b\n" 36 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) 37 : "r" (x) 38 : "memory"); 39 break; 40 case 2: 41 asm volatile("// __xchg2\n" 42 "1: ldxrh %w0, %2\n" 43 " stlxrh %w1, %w3, %2\n" 44 " cbnz %w1, 1b\n" 45 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr) 46 : "r" (x) 47 : "memory"); 48 break; 49 case 4: 50 asm volatile("// __xchg4\n" 51 "1: ldxr %w0, %2\n" 52 " stlxr %w1, %w3, %2\n" 53 " cbnz %w1, 1b\n" 54 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr) 55 : "r" (x) 56 : "memory"); 57 break; 58 case 8: 59 asm volatile("// __xchg8\n" 60 "1: ldxr %0, %2\n" 61 " stlxr %w1, %3, %2\n" 62 " cbnz %w1, 1b\n" 63 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr) 64 : "r" (x) 65 : "memory"); 66 break; 67 default: 68 BUILD_BUG(); 69 } 70 71 smp_mb(); 72 return ret; 73} 74 75#define xchg(ptr,x) \ 76({ \ 77 __typeof__(*(ptr)) __ret; \ 78 __ret = (__typeof__(*(ptr))) \ 79 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ 80 __ret; \ 81}) 82 83static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 84 unsigned long new, int size) 85{ 86 unsigned long oldval = 0, res; 87 88 switch (size) { 89 case 1: 90 do { 91 asm volatile("// __cmpxchg1\n" 92 " ldxrb %w1, %2\n" 93 " mov %w0, #0\n" 94 " cmp %w1, %w3\n" 95 " b.ne 1f\n" 96 " stxrb %w0, %w4, %2\n" 97 "1:\n" 98 : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr) 99 : "Ir" (old), "r" (new) 100 : "cc"); 101 } while (res); 102 break; 103 104 case 2: 105 do { 106 asm volatile("// __cmpxchg2\n" 107 " ldxrh %w1, %2\n" 108 " mov %w0, #0\n" 109 " cmp %w1, %w3\n" 110 " b.ne 1f\n" 111 " stxrh %w0, %w4, %2\n" 112 "1:\n" 113 : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr) 114 : "Ir" (old), "r" (new) 115 : "cc"); 116 } while (res); 117 break; 118 119 case 4: 120 do { 121 asm volatile("// __cmpxchg4\n" 122 " ldxr %w1, %2\n" 123 " mov %w0, #0\n" 124 " cmp %w1, %w3\n" 125 " b.ne 1f\n" 126 " stxr %w0, %w4, %2\n" 127 "1:\n" 128 : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr) 129 : "Ir" (old), "r" (new) 130 : "cc"); 131 } while (res); 132 break; 133 134 case 8: 135 do { 136 asm volatile("// __cmpxchg8\n" 137 " ldxr %1, %2\n" 138 " mov %w0, #0\n" 139 " cmp %1, %3\n" 140 " b.ne 1f\n" 141 " stxr %w0, %4, %2\n" 142 "1:\n" 143 : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr) 144 : "Ir" (old), "r" (new) 145 : "cc"); 146 } while (res); 147 break; 148 149 default: 150 BUILD_BUG(); 151 } 152 153 return oldval; 154} 155 156#define system_has_cmpxchg_double() 1 157 158static inline int __cmpxchg_double(volatile void *ptr1, volatile void *ptr2, 159 unsigned long old1, unsigned long old2, 160 unsigned long new1, unsigned long new2, int size) 161{ 162 unsigned long loop, lost; 163 164 switch (size) { 165 case 8: 166 VM_BUG_ON((unsigned long *)ptr2 - (unsigned long *)ptr1 != 1); 167 do { 168 asm volatile("// __cmpxchg_double8\n" 169 " ldxp %0, %1, %2\n" 170 " eor %0, %0, %3\n" 171 " eor %1, %1, %4\n" 172 " orr %1, %0, %1\n" 173 " mov %w0, #0\n" 174 " cbnz %1, 1f\n" 175 " stxp %w0, %5, %6, %2\n" 176 "1:\n" 177 : "=&r"(loop), "=&r"(lost), "+Q" (*(u64 *)ptr1) 178 : "r" (old1), "r"(old2), "r"(new1), "r"(new2)); 179 } while (loop); 180 break; 181 default: 182 BUILD_BUG(); 183 } 184 185 return !lost; 186} 187 188static inline int __cmpxchg_double_mb(volatile void *ptr1, volatile void *ptr2, 189 unsigned long old1, unsigned long old2, 190 unsigned long new1, unsigned long new2, int size) 191{ 192 int ret; 193 194 smp_mb(); 195 ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size); 196 smp_mb(); 197 198 return ret; 199} 200 201static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 202 unsigned long new, int size) 203{ 204 unsigned long ret; 205 206 smp_mb(); 207 ret = __cmpxchg(ptr, old, new, size); 208 smp_mb(); 209 210 return ret; 211} 212 213#define cmpxchg(ptr, o, n) \ 214({ \ 215 __typeof__(*(ptr)) __ret; \ 216 __ret = (__typeof__(*(ptr))) \ 217 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \ 218 sizeof(*(ptr))); \ 219 __ret; \ 220}) 221 222#define cmpxchg_local(ptr, o, n) \ 223({ \ 224 __typeof__(*(ptr)) __ret; \ 225 __ret = (__typeof__(*(ptr))) \ 226 __cmpxchg((ptr), (unsigned long)(o), \ 227 (unsigned long)(n), sizeof(*(ptr))); \ 228 __ret; \ 229}) 230 231#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \ 232({\ 233 int __ret;\ 234 __ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \ 235 (unsigned long)(o2), (unsigned long)(n1), \ 236 (unsigned long)(n2), sizeof(*(ptr1)));\ 237 __ret; \ 238}) 239 240#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \ 241({\ 242 int __ret;\ 243 __ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \ 244 (unsigned long)(o2), (unsigned long)(n1), \ 245 (unsigned long)(n2), sizeof(*(ptr1)));\ 246 __ret; \ 247}) 248 249#define _protect_cmpxchg_local(pcp, o, n) \ 250({ \ 251 typeof(*raw_cpu_ptr(&(pcp))) __ret; \ 252 preempt_disable(); \ 253 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ 254 preempt_enable(); \ 255 __ret; \ 256}) 257 258#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 259#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 260#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 261#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) 262 263#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ 264({ \ 265 int __ret; \ 266 preempt_disable(); \ 267 __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \ 268 raw_cpu_ptr(&(ptr2)), \ 269 o1, o2, n1, n2); \ 270 preempt_enable(); \ 271 __ret; \ 272}) 273 274#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) 275#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 276 277#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n)) 278 279#endif /* __ASM_CMPXCHG_H */ 280