1/*
2 *    Copyright IBM Corp. 1999,2013
3 *
4 *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
5 *
6 * The description below was taken in large parts from the powerpc
7 * bitops header file:
8 * Within a word, bits are numbered LSB first.  Lot's of places make
9 * this assumption by directly testing bits with (val & (1<<nr)).
10 * This can cause confusion for large (> 1 word) bitmaps on a
11 * big-endian system because, unlike little endian, the number of each
12 * bit depends on the word size.
13 *
14 * The bitop functions are defined to work on unsigned longs, so for an
15 * s390x system the bits end up numbered:
16 *   |63..............0|127............64|191...........128|255...........192|
17 * and on s390:
18 *   |31.....0|63....32|95....64|127...96|159..128|191..160|223..192|255..224|
19 *
20 * There are a few little-endian macros used mostly for filesystem
21 * bitmaps, these work on similar bit arrays layouts, but
22 * byte-oriented:
23 *   |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56|
24 *
25 * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit
26 * number field needs to be reversed compared to the big-endian bit
27 * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b).
28 *
29 * We also have special functions which work with an MSB0 encoding:
30 * on an s390x system the bits are numbered:
31 *   |0..............63|64............127|128...........191|192...........255|
32 * and on s390:
33 *   |0.....31|32....63|64....95|96...127|128..159|160..191|192..223|224..255|
34 *
35 * The main difference is that bit 0-63 (64b) or 0-31 (32b) in the bit
36 * number field needs to be reversed compared to the LSB0 encoded bit
37 * fields. This can be achieved by XOR with 0x3f (64b) or 0x1f (32b).
38 *
39 */
40
41#ifndef _S390_BITOPS_H
42#define _S390_BITOPS_H
43
44#ifndef _LINUX_BITOPS_H
45#error only <linux/bitops.h> can be included directly
46#endif
47
48#include <linux/typecheck.h>
49#include <linux/compiler.h>
50#include <asm/barrier.h>
51
52#define __BITOPS_NO_BARRIER	"\n"
53
54#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
55
56#define __BITOPS_OR		"laog"
57#define __BITOPS_AND		"lang"
58#define __BITOPS_XOR		"laxg"
59#define __BITOPS_BARRIER	"bcr	14,0\n"
60
61#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
62({								\
63	unsigned long __old;					\
64								\
65	typecheck(unsigned long *, (__addr));			\
66	asm volatile(						\
67		__barrier					\
68		__op_string "	%0,%2,%1\n"			\
69		__barrier					\
70		: "=d" (__old),	"+Q" (*(__addr))		\
71		: "d" (__val)					\
72		: "cc", "memory");				\
73	__old;							\
74})
75
76#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
77
78#define __BITOPS_OR		"ogr"
79#define __BITOPS_AND		"ngr"
80#define __BITOPS_XOR		"xgr"
81#define __BITOPS_BARRIER	"\n"
82
83#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
84({								\
85	unsigned long __old, __new;				\
86								\
87	typecheck(unsigned long *, (__addr));			\
88	asm volatile(						\
89		"	lg	%0,%2\n"			\
90		"0:	lgr	%1,%0\n"			\
91		__op_string "	%1,%3\n"			\
92		"	csg	%0,%1,%2\n"			\
93		"	jl	0b"				\
94		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
95		: "d" (__val)					\
96		: "cc", "memory");				\
97	__old;							\
98})
99
100#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
101
102#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
103
104static inline unsigned long *
105__bitops_word(unsigned long nr, volatile unsigned long *ptr)
106{
107	unsigned long addr;
108
109	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
110	return (unsigned long *)addr;
111}
112
113static inline unsigned char *
114__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
115{
116	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
117}
118
119static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
120{
121	unsigned long *addr = __bitops_word(nr, ptr);
122	unsigned long mask;
123
124#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
125	if (__builtin_constant_p(nr)) {
126		unsigned char *caddr = __bitops_byte(nr, ptr);
127
128		asm volatile(
129			"oi	%0,%b1\n"
130			: "+Q" (*caddr)
131			: "i" (1 << (nr & 7))
132			: "cc", "memory");
133		return;
134	}
135#endif
136	mask = 1UL << (nr & (BITS_PER_LONG - 1));
137	__BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);
138}
139
140static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
141{
142	unsigned long *addr = __bitops_word(nr, ptr);
143	unsigned long mask;
144
145#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
146	if (__builtin_constant_p(nr)) {
147		unsigned char *caddr = __bitops_byte(nr, ptr);
148
149		asm volatile(
150			"ni	%0,%b1\n"
151			: "+Q" (*caddr)
152			: "i" (~(1 << (nr & 7)))
153			: "cc", "memory");
154		return;
155	}
156#endif
157	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
158	__BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);
159}
160
161static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
162{
163	unsigned long *addr = __bitops_word(nr, ptr);
164	unsigned long mask;
165
166#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
167	if (__builtin_constant_p(nr)) {
168		unsigned char *caddr = __bitops_byte(nr, ptr);
169
170		asm volatile(
171			"xi	%0,%b1\n"
172			: "+Q" (*caddr)
173			: "i" (1 << (nr & 7))
174			: "cc", "memory");
175		return;
176	}
177#endif
178	mask = 1UL << (nr & (BITS_PER_LONG - 1));
179	__BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);
180}
181
182static inline int
183test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
184{
185	unsigned long *addr = __bitops_word(nr, ptr);
186	unsigned long old, mask;
187
188	mask = 1UL << (nr & (BITS_PER_LONG - 1));
189	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);
190	return (old & mask) != 0;
191}
192
193static inline int
194test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
195{
196	unsigned long *addr = __bitops_word(nr, ptr);
197	unsigned long old, mask;
198
199	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
200	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);
201	return (old & ~mask) != 0;
202}
203
204static inline int
205test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
206{
207	unsigned long *addr = __bitops_word(nr, ptr);
208	unsigned long old, mask;
209
210	mask = 1UL << (nr & (BITS_PER_LONG - 1));
211	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);
212	return (old & mask) != 0;
213}
214
215static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
216{
217	unsigned char *addr = __bitops_byte(nr, ptr);
218
219	*addr |= 1 << (nr & 7);
220}
221
222static inline void
223__clear_bit(unsigned long nr, volatile unsigned long *ptr)
224{
225	unsigned char *addr = __bitops_byte(nr, ptr);
226
227	*addr &= ~(1 << (nr & 7));
228}
229
230static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
231{
232	unsigned char *addr = __bitops_byte(nr, ptr);
233
234	*addr ^= 1 << (nr & 7);
235}
236
237static inline int
238__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
239{
240	unsigned char *addr = __bitops_byte(nr, ptr);
241	unsigned char ch;
242
243	ch = *addr;
244	*addr |= 1 << (nr & 7);
245	return (ch >> (nr & 7)) & 1;
246}
247
248static inline int
249__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
250{
251	unsigned char *addr = __bitops_byte(nr, ptr);
252	unsigned char ch;
253
254	ch = *addr;
255	*addr &= ~(1 << (nr & 7));
256	return (ch >> (nr & 7)) & 1;
257}
258
259static inline int
260__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
261{
262	unsigned char *addr = __bitops_byte(nr, ptr);
263	unsigned char ch;
264
265	ch = *addr;
266	*addr ^= 1 << (nr & 7);
267	return (ch >> (nr & 7)) & 1;
268}
269
270static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
271{
272	const volatile unsigned char *addr;
273
274	addr = ((const volatile unsigned char *)ptr);
275	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
276	return (*addr >> (nr & 7)) & 1;
277}
278
279/*
280 * Functions which use MSB0 bit numbering.
281 * On an s390x system the bits are numbered:
282 *   |0..............63|64............127|128...........191|192...........255|
283 * and on s390:
284 *   |0.....31|32....63|64....95|96...127|128..159|160..191|192..223|224..255|
285 */
286unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
287unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
288				unsigned long offset);
289
290static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
291{
292	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
293}
294
295static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
296{
297	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
298}
299
300static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
301{
302	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
303}
304
305static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
306{
307	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
308}
309
310static inline int test_bit_inv(unsigned long nr,
311			       const volatile unsigned long *ptr)
312{
313	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
314}
315
316#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
317
318/**
319 * __flogr - find leftmost one
320 * @word - The word to search
321 *
322 * Returns the bit number of the most significant bit set,
323 * where the most significant bit has bit number 0.
324 * If no bit is set this function returns 64.
325 */
326static inline unsigned char __flogr(unsigned long word)
327{
328	if (__builtin_constant_p(word)) {
329		unsigned long bit = 0;
330
331		if (!word)
332			return 64;
333		if (!(word & 0xffffffff00000000UL)) {
334			word <<= 32;
335			bit += 32;
336		}
337		if (!(word & 0xffff000000000000UL)) {
338			word <<= 16;
339			bit += 16;
340		}
341		if (!(word & 0xff00000000000000UL)) {
342			word <<= 8;
343			bit += 8;
344		}
345		if (!(word & 0xf000000000000000UL)) {
346			word <<= 4;
347			bit += 4;
348		}
349		if (!(word & 0xc000000000000000UL)) {
350			word <<= 2;
351			bit += 2;
352		}
353		if (!(word & 0x8000000000000000UL)) {
354			word <<= 1;
355			bit += 1;
356		}
357		return bit;
358	} else {
359		register unsigned long bit asm("4") = word;
360		register unsigned long out asm("5");
361
362		asm volatile(
363			"       flogr   %[bit],%[bit]\n"
364			: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
365		return bit;
366	}
367}
368
369/**
370 * __ffs - find first bit in word.
371 * @word: The word to search
372 *
373 * Undefined if no bit exists, so code should check against 0 first.
374 */
375static inline unsigned long __ffs(unsigned long word)
376{
377	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
378}
379
380/**
381 * ffs - find first bit set
382 * @word: the word to search
383 *
384 * This is defined the same way as the libc and
385 * compiler builtin ffs routines (man ffs).
386 */
387static inline int ffs(int word)
388{
389	unsigned long mask = 2 * BITS_PER_LONG - 1;
390	unsigned int val = (unsigned int)word;
391
392	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
393}
394
395/**
396 * __fls - find last (most-significant) set bit in a long word
397 * @word: the word to search
398 *
399 * Undefined if no set bit exists, so code should check against 0 first.
400 */
401static inline unsigned long __fls(unsigned long word)
402{
403	return __flogr(word) ^ (BITS_PER_LONG - 1);
404}
405
406/**
407 * fls64 - find last set bit in a 64-bit word
408 * @word: the word to search
409 *
410 * This is defined in a similar way as the libc and compiler builtin
411 * ffsll, but returns the position of the most significant set bit.
412 *
413 * fls64(value) returns 0 if value is 0 or the position of the last
414 * set bit if value is nonzero. The last (most significant) bit is
415 * at position 64.
416 */
417static inline int fls64(unsigned long word)
418{
419	unsigned long mask = 2 * BITS_PER_LONG - 1;
420
421	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
422}
423
424/**
425 * fls - find last (most-significant) bit set
426 * @word: the word to search
427 *
428 * This is defined the same way as ffs.
429 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
430 */
431static inline int fls(int word)
432{
433	return fls64((unsigned int)word);
434}
435
436#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
437
438#include <asm-generic/bitops/__ffs.h>
439#include <asm-generic/bitops/ffs.h>
440#include <asm-generic/bitops/__fls.h>
441#include <asm-generic/bitops/fls.h>
442#include <asm-generic/bitops/fls64.h>
443
444#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
445
446#include <asm-generic/bitops/ffz.h>
447#include <asm-generic/bitops/find.h>
448#include <asm-generic/bitops/hweight.h>
449#include <asm-generic/bitops/lock.h>
450#include <asm-generic/bitops/sched.h>
451#include <asm-generic/bitops/le.h>
452#include <asm-generic/bitops/ext2-atomic-setbit.h>
453
454#endif /* _S390_BITOPS_H */
455