Branch data Line data Source code
1 : : /*
2 : : * Copyright 1995, Russell King.
3 : : * Various bits and pieces copyrights include:
4 : : * Linus Torvalds (test_bit).
5 : : * Big endian support: Copyright 2001, Nicolas Pitre
6 : : * reworked by rmk.
7 : : *
8 : : * bit 0 is the LSB of an "unsigned long" quantity.
9 : : *
10 : : * Please note that the code in this file should never be included
11 : : * from user space. Many of these are not implemented in assembler
12 : : * since they would be too costly. Also, they require privileged
13 : : * instructions (which are not available from user mode) to ensure
14 : : * that they are atomic.
15 : : */
16 : :
17 : : #ifndef __ASM_ARM_BITOPS_H
18 : : #define __ASM_ARM_BITOPS_H
19 : :
20 : : #ifdef __KERNEL__
21 : :
22 : : #ifndef _LINUX_BITOPS_H
23 : : #error only <linux/bitops.h> can be included directly
24 : : #endif
25 : :
26 : : #include <linux/compiler.h>
27 : : #include <linux/irqflags.h>
28 : :
29 : : #define smp_mb__before_clear_bit() smp_mb()
30 : : #define smp_mb__after_clear_bit() smp_mb()
31 : :
32 : : /*
33 : : * These functions are the basis of our bit ops.
34 : : *
35 : : * First, the atomic bitops. These use native endian.
36 : : */
37 : : static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *p)
38 : : {
39 : : unsigned long flags;
40 : : unsigned long mask = 1UL << (bit & 31);
41 : :
42 : : p += bit >> 5;
43 : :
44 : : raw_local_irq_save(flags);
45 : : *p |= mask;
46 : : raw_local_irq_restore(flags);
47 : : }
48 : :
49 : : static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
50 : : {
51 : : unsigned long flags;
52 : : unsigned long mask = 1UL << (bit & 31);
53 : :
54 : : p += bit >> 5;
55 : :
56 : : raw_local_irq_save(flags);
57 : : *p &= ~mask;
58 : : raw_local_irq_restore(flags);
59 : : }
60 : :
61 : : static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
62 : : {
63 : : unsigned long flags;
64 : : unsigned long mask = 1UL << (bit & 31);
65 : :
66 : : p += bit >> 5;
67 : :
68 : : raw_local_irq_save(flags);
69 : : *p ^= mask;
70 : : raw_local_irq_restore(flags);
71 : : }
72 : :
73 : : static inline int
74 : : ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
75 : : {
76 : : unsigned long flags;
77 : : unsigned int res;
78 : : unsigned long mask = 1UL << (bit & 31);
79 : :
80 : : p += bit >> 5;
81 : :
82 : : raw_local_irq_save(flags);
83 : : res = *p;
84 : : *p = res | mask;
85 : : raw_local_irq_restore(flags);
86 : :
87 : : return (res & mask) != 0;
88 : : }
89 : :
90 : : static inline int
91 : : ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
92 : : {
93 : : unsigned long flags;
94 : : unsigned int res;
95 : : unsigned long mask = 1UL << (bit & 31);
96 : :
97 : : p += bit >> 5;
98 : :
99 : : raw_local_irq_save(flags);
100 : : res = *p;
101 : : *p = res & ~mask;
102 : : raw_local_irq_restore(flags);
103 : :
104 : : return (res & mask) != 0;
105 : : }
106 : :
107 : : static inline int
108 : : ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
109 : : {
110 : : unsigned long flags;
111 : : unsigned int res;
112 : : unsigned long mask = 1UL << (bit & 31);
113 : :
114 : : p += bit >> 5;
115 : :
116 : : raw_local_irq_save(flags);
117 : : res = *p;
118 : : *p = res ^ mask;
119 : : raw_local_irq_restore(flags);
120 : :
121 : : return (res & mask) != 0;
122 : : }
123 : :
124 : : #include <asm-generic/bitops/non-atomic.h>
125 : :
126 : : /*
127 : : * A note about Endian-ness.
128 : : * -------------------------
129 : : *
130 : : * When the ARM is put into big endian mode via CR15, the processor
131 : : * merely swaps the order of bytes within words, thus:
132 : : *
133 : : * ------------ physical data bus bits -----------
134 : : * D31 ... D24 D23 ... D16 D15 ... D8 D7 ... D0
135 : : * little byte 3 byte 2 byte 1 byte 0
136 : : * big byte 0 byte 1 byte 2 byte 3
137 : : *
138 : : * This means that reading a 32-bit word at address 0 returns the same
139 : : * value irrespective of the endian mode bit.
140 : : *
141 : : * Peripheral devices should be connected with the data bus reversed in
142 : : * "Big Endian" mode. ARM Application Note 61 is applicable, and is
143 : : * available from http://www.arm.com/.
144 : : *
145 : : * The following assumes that the data bus connectivity for big endian
146 : : * mode has been followed.
147 : : *
148 : : * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0.
149 : : */
150 : :
151 : : /*
152 : : * Native endian assembly bitops. nr = 0 -> word 0 bit 0.
153 : : */
154 : : extern void _set_bit(int nr, volatile unsigned long * p);
155 : : extern void _clear_bit(int nr, volatile unsigned long * p);
156 : : extern void _change_bit(int nr, volatile unsigned long * p);
157 : : extern int _test_and_set_bit(int nr, volatile unsigned long * p);
158 : : extern int _test_and_clear_bit(int nr, volatile unsigned long * p);
159 : : extern int _test_and_change_bit(int nr, volatile unsigned long * p);
160 : :
161 : : /*
162 : : * Little endian assembly bitops. nr = 0 -> byte 0 bit 0.
163 : : */
164 : : extern int _find_first_zero_bit_le(const void * p, unsigned size);
165 : : extern int _find_next_zero_bit_le(const void * p, int size, int offset);
166 : : extern int _find_first_bit_le(const unsigned long *p, unsigned size);
167 : : extern int _find_next_bit_le(const unsigned long *p, int size, int offset);
168 : :
169 : : /*
170 : : * Big endian assembly bitops. nr = 0 -> byte 3 bit 0.
171 : : */
172 : : extern int _find_first_zero_bit_be(const void * p, unsigned size);
173 : : extern int _find_next_zero_bit_be(const void * p, int size, int offset);
174 : : extern int _find_first_bit_be(const unsigned long *p, unsigned size);
175 : : extern int _find_next_bit_be(const unsigned long *p, int size, int offset);
176 : :
177 : : #ifndef CONFIG_SMP
178 : : /*
179 : : * The __* form of bitops are non-atomic and may be reordered.
180 : : */
181 : : #define ATOMIC_BITOP(name,nr,p) \
182 : : (__builtin_constant_p(nr) ? ____atomic_##name(nr, p) : _##name(nr,p))
183 : : #else
184 : : #define ATOMIC_BITOP(name,nr,p) _##name(nr,p)
185 : : #endif
186 : :
187 : : /*
188 : : * Native endian atomic definitions.
189 : : */
190 : : #define set_bit(nr,p) ATOMIC_BITOP(set_bit,nr,p)
191 : : #define clear_bit(nr,p) ATOMIC_BITOP(clear_bit,nr,p)
192 : : #define change_bit(nr,p) ATOMIC_BITOP(change_bit,nr,p)
193 : : #define test_and_set_bit(nr,p) ATOMIC_BITOP(test_and_set_bit,nr,p)
194 : : #define test_and_clear_bit(nr,p) ATOMIC_BITOP(test_and_clear_bit,nr,p)
195 : : #define test_and_change_bit(nr,p) ATOMIC_BITOP(test_and_change_bit,nr,p)
196 : :
197 : : #ifndef __ARMEB__
198 : : /*
199 : : * These are the little endian, atomic definitions.
200 : : */
201 : : #define find_first_zero_bit(p,sz) _find_first_zero_bit_le(p,sz)
202 : : #define find_next_zero_bit(p,sz,off) _find_next_zero_bit_le(p,sz,off)
203 : : #define find_first_bit(p,sz) _find_first_bit_le(p,sz)
204 : : #define find_next_bit(p,sz,off) _find_next_bit_le(p,sz,off)
205 : :
206 : : #else
207 : : /*
208 : : * These are the big endian, atomic definitions.
209 : : */
210 : : #define find_first_zero_bit(p,sz) _find_first_zero_bit_be(p,sz)
211 : : #define find_next_zero_bit(p,sz,off) _find_next_zero_bit_be(p,sz,off)
212 : : #define find_first_bit(p,sz) _find_first_bit_be(p,sz)
213 : : #define find_next_bit(p,sz,off) _find_next_bit_be(p,sz,off)
214 : :
215 : : #endif
216 : :
217 : : #if __LINUX_ARM_ARCH__ < 5
218 : :
219 : : #include <asm-generic/bitops/ffz.h>
220 : : #include <asm-generic/bitops/__fls.h>
221 : : #include <asm-generic/bitops/__ffs.h>
222 : : #include <asm-generic/bitops/fls.h>
223 : : #include <asm-generic/bitops/ffs.h>
224 : :
225 : : #else
226 : :
227 : : static inline int constant_fls(int x)
228 : : {
229 : : int r = 32;
230 : :
231 [ - + ][ - + ]: 13752999 : if (!x)
[ - + ][ - + ]
[ - - ][ - - ]
[ # # ][ # # ]
[ - + ][ - + ]
[ # # ][ - + ]
[ # # ][ # # ]
[ - ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ - ]
[ - ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ - ][ - ]
232 : : return 0;
233 [ # # ][ # # ]: 58985 : if (!(x & 0xffff0000u)) {
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
234 : 0 : x <<= 16;
235 : : r -= 16;
236 : : }
237 [ # # ][ # # ]: 0 : if (!(x & 0xff000000u)) {
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
238 : 0 : x <<= 8;
239 : 0 : r -= 8;
240 : : }
241 [ # # ][ # # ]: 0 : if (!(x & 0xf0000000u)) {
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
242 : 0 : x <<= 4;
243 : 0 : r -= 4;
244 : : }
245 [ # # ][ # # ]: 0 : if (!(x & 0xc0000000u)) {
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ - ]
246 : 0 : x <<= 2;
247 : 0 : r -= 2;
248 : : }
249 [ # # ][ # # ]: 0 : if (!(x & 0x80000000u)) {
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ - ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ - ]
[ - ]
250 : : x <<= 1;
251 : 0 : r -= 1;
252 : : }
253 : : return r;
254 : : }
255 : :
256 : : /*
257 : : * On ARMv5 and above those functions can be implemented around
258 : : * the clz instruction for much better code efficiency.
259 : : */
260 : :
261 : : static inline int fls(int x)
262 : : {
263 : : int ret;
264 : :
265 [ + + ][ + + ]: 126976347 : if (__builtin_constant_p(x))
[ + + ][ + + ]
[ + + ][ + + ]
[ - + ][ - + ]
[ + + ][ + + ]
[ # + ][ + + ]
[ - + ][ - + ]
[ # # ][ # # ]
[ # # ][ # # ]
[ - + ][ - + ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ][ # # ]
266 : : return constant_fls(x);
267 : :
268 : 95494932 : asm("clz\t%0, %1" : "=r" (ret) : "r" (x));
269 : 95494937 : ret = 32 - ret;
270 : : return ret;
271 : : }
272 : :
273 : : #define __fls(x) (fls(x) - 1)
274 : : #define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); })
275 : : #define __ffs(x) (ffs(x) - 1)
276 : : #define ffz(x) __ffs( ~(x) )
277 : :
278 : : #endif
279 : :
280 : : #include <asm-generic/bitops/fls64.h>
281 : :
282 : : #include <asm-generic/bitops/sched.h>
283 : : #include <asm-generic/bitops/hweight.h>
284 : : #include <asm-generic/bitops/lock.h>
285 : :
286 : : #ifdef __ARMEB__
287 : :
288 : : static inline int find_first_zero_bit_le(const void *p, unsigned size)
289 : : {
290 : : return _find_first_zero_bit_le(p, size);
291 : : }
292 : : #define find_first_zero_bit_le find_first_zero_bit_le
293 : :
294 : : static inline int find_next_zero_bit_le(const void *p, int size, int offset)
295 : : {
296 : : return _find_next_zero_bit_le(p, size, offset);
297 : : }
298 : : #define find_next_zero_bit_le find_next_zero_bit_le
299 : :
300 : : static inline int find_next_bit_le(const void *p, int size, int offset)
301 : : {
302 : : return _find_next_bit_le(p, size, offset);
303 : : }
304 : : #define find_next_bit_le find_next_bit_le
305 : :
306 : : #endif
307 : :
308 : : #include <asm-generic/bitops/le.h>
309 : :
310 : : /*
311 : : * Ext2 is defined to use little-endian byte ordering.
312 : : */
313 : : #include <asm-generic/bitops/ext2-atomic-setbit.h>
314 : :
315 : : #endif /* __KERNEL__ */
316 : :
317 : : #endif /* _ARM_BITOPS_H */
|
