1 /*
2 * User address space access functions.
3 * The non-inlined parts of asm-metag/uaccess.h are here.
4 *
5 * Copyright (C) 2006, Imagination Technologies.
6 * Copyright (C) 2000, Axis Communications AB.
7 *
8 * Written by Hans-Peter Nilsson.
9 * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
10 * Modified for Meta by Will Newton.
11 */
12
13 #include <linux/export.h>
14 #include <linux/uaccess.h>
15 #include <asm/cache.h> /* def of L1_CACHE_BYTES */
16
17 #define USE_RAPF
18 #define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES)
19
20
21 /* The "double write" in this code is because the Meta will not fault
22 * immediately unless the memory pipe is forced to by e.g. a data stall or
23 * another memory op. The second write should be discarded by the write
24 * combiner so should have virtually no cost.
25 */
26
27 #define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \
28 asm volatile ( \
29 COPY \
30 "1:\n" \
31 " .section .fixup,\"ax\"\n" \
32 " MOV D1Ar1,#0\n" \
33 FIXUP \
34 " MOVT D1Ar1,#HI(1b)\n" \
35 " JUMP D1Ar1,#LO(1b)\n" \
36 " .previous\n" \
37 " .section __ex_table,\"a\"\n" \
38 TENTRY \
39 " .previous\n" \
40 : "=r" (to), "=r" (from), "=r" (ret) \
41 : "0" (to), "1" (from), "2" (ret) \
42 : "D1Ar1", "memory")
43
44
45 #define __asm_copy_to_user_1(to, from, ret) \
46 __asm_copy_user_cont(to, from, ret, \
47 " GETB D1Ar1,[%1++]\n" \
48 " SETB [%0],D1Ar1\n" \
49 "2: SETB [%0++],D1Ar1\n", \
50 "3: ADD %2,%2,#1\n", \
51 " .long 2b,3b\n")
52
53 #define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
54 __asm_copy_user_cont(to, from, ret, \
55 " GETW D1Ar1,[%1++]\n" \
56 " SETW [%0],D1Ar1\n" \
57 "2: SETW [%0++],D1Ar1\n" COPY, \
58 "3: ADD %2,%2,#2\n" FIXUP, \
59 " .long 2b,3b\n" TENTRY)
60
61 #define __asm_copy_to_user_2(to, from, ret) \
62 __asm_copy_to_user_2x_cont(to, from, ret, "", "", "")
63
64 #define __asm_copy_to_user_3(to, from, ret) \
65 __asm_copy_to_user_2x_cont(to, from, ret, \
66 " GETB D1Ar1,[%1++]\n" \
67 " SETB [%0],D1Ar1\n" \
68 "4: SETB [%0++],D1Ar1\n", \
69 "5: ADD %2,%2,#1\n", \
70 " .long 4b,5b\n")
71
72 #define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
73 __asm_copy_user_cont(to, from, ret, \
74 " GETD D1Ar1,[%1++]\n" \
75 " SETD [%0],D1Ar1\n" \
76 "2: SETD [%0++],D1Ar1\n" COPY, \
77 "3: ADD %2,%2,#4\n" FIXUP, \
78 " .long 2b,3b\n" TENTRY)
79
80 #define __asm_copy_to_user_4(to, from, ret) \
81 __asm_copy_to_user_4x_cont(to, from, ret, "", "", "")
82
83 #define __asm_copy_to_user_5(to, from, ret) \
84 __asm_copy_to_user_4x_cont(to, from, ret, \
85 " GETB D1Ar1,[%1++]\n" \
86 " SETB [%0],D1Ar1\n" \
87 "4: SETB [%0++],D1Ar1\n", \
88 "5: ADD %2,%2,#1\n", \
89 " .long 4b,5b\n")
90
91 #define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
92 __asm_copy_to_user_4x_cont(to, from, ret, \
93 " GETW D1Ar1,[%1++]\n" \
94 " SETW [%0],D1Ar1\n" \
95 "4: SETW [%0++],D1Ar1\n" COPY, \
96 "5: ADD %2,%2,#2\n" FIXUP, \
97 " .long 4b,5b\n" TENTRY)
98
99 #define __asm_copy_to_user_6(to, from, ret) \
100 __asm_copy_to_user_6x_cont(to, from, ret, "", "", "")
101
102 #define __asm_copy_to_user_7(to, from, ret) \
103 __asm_copy_to_user_6x_cont(to, from, ret, \
104 " GETB D1Ar1,[%1++]\n" \
105 " SETB [%0],D1Ar1\n" \
106 "6: SETB [%0++],D1Ar1\n", \
107 "7: ADD %2,%2,#1\n", \
108 " .long 6b,7b\n")
109
110 #define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
111 __asm_copy_to_user_4x_cont(to, from, ret, \
112 " GETD D1Ar1,[%1++]\n" \
113 " SETD [%0],D1Ar1\n" \
114 "4: SETD [%0++],D1Ar1\n" COPY, \
115 "5: ADD %2,%2,#4\n" FIXUP, \
116 " .long 4b,5b\n" TENTRY)
117
118 #define __asm_copy_to_user_8(to, from, ret) \
119 __asm_copy_to_user_8x_cont(to, from, ret, "", "", "")
120
121 #define __asm_copy_to_user_9(to, from, ret) \
122 __asm_copy_to_user_8x_cont(to, from, ret, \
123 " GETB D1Ar1,[%1++]\n" \
124 " SETB [%0],D1Ar1\n" \
125 "6: SETB [%0++],D1Ar1\n", \
126 "7: ADD %2,%2,#1\n", \
127 " .long 6b,7b\n")
128
129 #define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
130 __asm_copy_to_user_8x_cont(to, from, ret, \
131 " GETW D1Ar1,[%1++]\n" \
132 " SETW [%0],D1Ar1\n" \
133 "6: SETW [%0++],D1Ar1\n" COPY, \
134 "7: ADD %2,%2,#2\n" FIXUP, \
135 " .long 6b,7b\n" TENTRY)
136
137 #define __asm_copy_to_user_10(to, from, ret) \
138 __asm_copy_to_user_10x_cont(to, from, ret, "", "", "")
139
140 #define __asm_copy_to_user_11(to, from, ret) \
141 __asm_copy_to_user_10x_cont(to, from, ret, \
142 " GETB D1Ar1,[%1++]\n" \
143 " SETB [%0],D1Ar1\n" \
144 "8: SETB [%0++],D1Ar1\n", \
145 "9: ADD %2,%2,#1\n", \
146 " .long 8b,9b\n")
147
148 #define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
149 __asm_copy_to_user_8x_cont(to, from, ret, \
150 " GETD D1Ar1,[%1++]\n" \
151 " SETD [%0],D1Ar1\n" \
152 "6: SETD [%0++],D1Ar1\n" COPY, \
153 "7: ADD %2,%2,#4\n" FIXUP, \
154 " .long 6b,7b\n" TENTRY)
155 #define __asm_copy_to_user_12(to, from, ret) \
156 __asm_copy_to_user_12x_cont(to, from, ret, "", "", "")
157
158 #define __asm_copy_to_user_13(to, from, ret) \
159 __asm_copy_to_user_12x_cont(to, from, ret, \
160 " GETB D1Ar1,[%1++]\n" \
161 " SETB [%0],D1Ar1\n" \
162 "8: SETB [%0++],D1Ar1\n", \
163 "9: ADD %2,%2,#1\n", \
164 " .long 8b,9b\n")
165
166 #define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
167 __asm_copy_to_user_12x_cont(to, from, ret, \
168 " GETW D1Ar1,[%1++]\n" \
169 " SETW [%0],D1Ar1\n" \
170 "8: SETW [%0++],D1Ar1\n" COPY, \
171 "9: ADD %2,%2,#2\n" FIXUP, \
172 " .long 8b,9b\n" TENTRY)
173
174 #define __asm_copy_to_user_14(to, from, ret) \
175 __asm_copy_to_user_14x_cont(to, from, ret, "", "", "")
176
177 #define __asm_copy_to_user_15(to, from, ret) \
178 __asm_copy_to_user_14x_cont(to, from, ret, \
179 " GETB D1Ar1,[%1++]\n" \
180 " SETB [%0],D1Ar1\n" \
181 "10: SETB [%0++],D1Ar1\n", \
182 "11: ADD %2,%2,#1\n", \
183 " .long 10b,11b\n")
184
185 #define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
186 __asm_copy_to_user_12x_cont(to, from, ret, \
187 " GETD D1Ar1,[%1++]\n" \
188 " SETD [%0],D1Ar1\n" \
189 "8: SETD [%0++],D1Ar1\n" COPY, \
190 "9: ADD %2,%2,#4\n" FIXUP, \
191 " .long 8b,9b\n" TENTRY)
192
193 #define __asm_copy_to_user_16(to, from, ret) \
194 __asm_copy_to_user_16x_cont(to, from, ret, "", "", "")
195
196 #define __asm_copy_to_user_8x64(to, from, ret) \
197 asm volatile ( \
198 " GETL D0Ar2,D1Ar1,[%1++]\n" \
199 " SETL [%0],D0Ar2,D1Ar1\n" \
200 "2: SETL [%0++],D0Ar2,D1Ar1\n" \
201 "1:\n" \
202 " .section .fixup,\"ax\"\n" \
203 "3: ADD %2,%2,#8\n" \
204 " MOVT D0Ar2,#HI(1b)\n" \
205 " JUMP D0Ar2,#LO(1b)\n" \
206 " .previous\n" \
207 " .section __ex_table,\"a\"\n" \
208 " .long 2b,3b\n" \
209 " .previous\n" \
210 : "=r" (to), "=r" (from), "=r" (ret) \
211 : "0" (to), "1" (from), "2" (ret) \
212 : "D1Ar1", "D0Ar2", "memory")
213
214 /*
215 * optimized copying loop using RAPF when 64 bit aligned
216 *
217 * n will be automatically decremented inside the loop
218 * ret will be left intact. if error occurs we will rewind
219 * so that the original non optimized code will fill up
220 * this value correctly.
221 *
222 * on fault:
223 * > n will hold total number of uncopied bytes
224 *
225 * > {'to','from'} will be rewind back so that
226 * the non-optimized code will do the proper fix up
227 *
228 * DCACHE drops the cacheline which helps in reducing cache
229 * pollution.
230 *
231 * We introduce an extra SETL at the end of the loop to
232 * ensure we don't fall off the loop before we catch all
233 * erros.
234 *
235 * NOTICE:
236 * LSM_STEP in TXSTATUS must be cleared in fix up code.
237 * since we're using M{S,G}ETL, a fault might happen at
238 * any address in the middle of M{S,G}ETL causing
239 * the value of LSM_STEP to be incorrect which can
240 * cause subsequent use of M{S,G}ET{L,D} to go wrong.
241 * ie: if LSM_STEP was 1 when a fault occurs, the
242 * next call to M{S,G}ET{L,D} will skip the first
243 * copy/getting as it think that the first 1 has already
244 * been done.
245 *
246 */
247 #define __asm_copy_user_64bit_rapf_loop( \
248 to, from, ret, n, id, FIXUP) \
249 asm volatile ( \
250 ".balign 8\n" \
251 "MOV RAPF, %1\n" \
252 "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
253 "MOV D0Ar6, #0\n" \
254 "LSR D1Ar5, %3, #6\n" \
255 "SUB TXRPT, D1Ar5, #2\n" \
256 "MOV RAPF, %1\n" \
257 "$Lloop"id":\n" \
258 "ADD RAPF, %1, #64\n" \
259 "21:\n" \
260 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
261 "22:\n" \
262 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
263 "SUB %3, %3, #32\n" \
264 "23:\n" \
265 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
266 "24:\n" \
267 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
268 "SUB %3, %3, #32\n" \
269 "DCACHE [%1+#-64], D0Ar6\n" \
270 "BR $Lloop"id"\n" \
271 \
272 "MOV RAPF, %1\n" \
273 "25:\n" \
274 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
275 "26:\n" \
276 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
277 "SUB %3, %3, #32\n" \
278 "27:\n" \
279 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
280 "28:\n" \
281 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
282 "SUB %0, %0, #8\n" \
283 "29:\n" \
284 "SETL [%0++], D0.7, D1.7\n" \
285 "SUB %3, %3, #32\n" \
286 "1:" \
287 "DCACHE [%1+#-64], D0Ar6\n" \
288 "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \
289 "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \
290 "GETL D0.5, D1.5, [A0StP+#-24]\n" \
291 "GETL D0.6, D1.6, [A0StP+#-16]\n" \
292 "GETL D0.7, D1.7, [A0StP+#-8]\n" \
293 "SUB A0StP, A0StP, #40\n" \
294 " .section .fixup,\"ax\"\n" \
295 "4:\n" \
296 " ADD %0, %0, #8\n" \
297 "3:\n" \
298 " MOV D0Ar2, TXSTATUS\n" \
299 " MOV D1Ar1, TXSTATUS\n" \
300 " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \
301 " MOV TXSTATUS, D1Ar1\n" \
302 FIXUP \
303 " MOVT D0Ar2,#HI(1b)\n" \
304 " JUMP D0Ar2,#LO(1b)\n" \
305 " .previous\n" \
306 " .section __ex_table,\"a\"\n" \
307 " .long 21b,3b\n" \
308 " .long 22b,3b\n" \
309 " .long 23b,3b\n" \
310 " .long 24b,3b\n" \
311 " .long 25b,3b\n" \
312 " .long 26b,3b\n" \
313 " .long 27b,3b\n" \
314 " .long 28b,3b\n" \
315 " .long 29b,4b\n" \
316 " .previous\n" \
317 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
318 : "0" (to), "1" (from), "2" (ret), "3" (n) \
319 : "D1Ar1", "D0Ar2", "memory")
320
321 /* rewind 'to' and 'from' pointers when a fault occurs
322 *
323 * Rationale:
324 * A fault always occurs on writing to user buffer. A fault
325 * is at a single address, so we need to rewind by only 4
326 * bytes.
327 * Since we do a complete read from kernel buffer before
328 * writing, we need to rewind it also. The amount to be
329 * rewind equals the number of faulty writes in MSETD
330 * which is: [4 - (LSM_STEP-1)]*8
331 * LSM_STEP is bits 10:8 in TXSTATUS which is already read
332 * and stored in D0Ar2
333 *
334 * NOTE: If a fault occurs at the last operation in M{G,S}ETL
335 * LSM_STEP will be 0. ie: we do 4 writes in our case, if
336 * a fault happens at the 4th write, LSM_STEP will be 0
337 * instead of 4. The code copes with that.
338 *
339 * n is updated by the number of successful writes, which is:
340 * n = n - (LSM_STEP-1)*8
341 */
342 #define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\
343 __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
344 "LSR D0Ar2, D0Ar2, #8\n" \
345 "AND D0Ar2, D0Ar2, #0x7\n" \
346 "ADDZ D0Ar2, D0Ar2, #4\n" \
347 "SUB D0Ar2, D0Ar2, #1\n" \
348 "MOV D1Ar1, #4\n" \
349 "SUB D0Ar2, D1Ar1, D0Ar2\n" \
350 "LSL D0Ar2, D0Ar2, #3\n" \
351 "LSL D1Ar1, D1Ar1, #3\n" \
352 "SUB D1Ar1, D1Ar1, D0Ar2\n" \
353 "SUB %0, %0, #8\n" \
354 "SUB %1, %1,D0Ar2\n" \
355 "SUB %3, %3, D1Ar1\n")
356
357 /*
358 * optimized copying loop using RAPF when 32 bit aligned
359 *
360 * n will be automatically decremented inside the loop
361 * ret will be left intact. if error occurs we will rewind
362 * so that the original non optimized code will fill up
363 * this value correctly.
364 *
365 * on fault:
366 * > n will hold total number of uncopied bytes
367 *
368 * > {'to','from'} will be rewind back so that
369 * the non-optimized code will do the proper fix up
370 *
371 * DCACHE drops the cacheline which helps in reducing cache
372 * pollution.
373 *
374 * We introduce an extra SETD at the end of the loop to
375 * ensure we don't fall off the loop before we catch all
376 * erros.
377 *
378 * NOTICE:
379 * LSM_STEP in TXSTATUS must be cleared in fix up code.
380 * since we're using M{S,G}ETL, a fault might happen at
381 * any address in the middle of M{S,G}ETL causing
382 * the value of LSM_STEP to be incorrect which can
383 * cause subsequent use of M{S,G}ET{L,D} to go wrong.
384 * ie: if LSM_STEP was 1 when a fault occurs, the
385 * next call to M{S,G}ET{L,D} will skip the first
386 * copy/getting as it think that the first 1 has already
387 * been done.
388 *
389 */
390 #define __asm_copy_user_32bit_rapf_loop( \
391 to, from, ret, n, id, FIXUP) \
392 asm volatile ( \
393 ".balign 8\n" \
394 "MOV RAPF, %1\n" \
395 "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
396 "MOV D0Ar6, #0\n" \
397 "LSR D1Ar5, %3, #6\n" \
398 "SUB TXRPT, D1Ar5, #2\n" \
399 "MOV RAPF, %1\n" \
400 "$Lloop"id":\n" \
401 "ADD RAPF, %1, #64\n" \
402 "21:\n" \
403 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
404 "22:\n" \
405 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
406 "SUB %3, %3, #16\n" \
407 "23:\n" \
408 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
409 "24:\n" \
410 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
411 "SUB %3, %3, #16\n" \
412 "25:\n" \
413 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
414 "26:\n" \
415 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
416 "SUB %3, %3, #16\n" \
417 "27:\n" \
418 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
419 "28:\n" \
420 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
421 "SUB %3, %3, #16\n" \
422 "DCACHE [%1+#-64], D0Ar6\n" \
423 "BR $Lloop"id"\n" \
424 \
425 "MOV RAPF, %1\n" \
426 "29:\n" \
427 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
428 "30:\n" \
429 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
430 "SUB %3, %3, #16\n" \
431 "31:\n" \
432 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
433 "32:\n" \
434 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
435 "SUB %3, %3, #16\n" \
436 "33:\n" \
437 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
438 "34:\n" \
439 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
440 "SUB %3, %3, #16\n" \
441 "35:\n" \
442 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
443 "36:\n" \
444 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
445 "SUB %0, %0, #4\n" \
446 "37:\n" \
447 "SETD [%0++], D0.7\n" \
448 "SUB %3, %3, #16\n" \
449 "1:" \
450 "DCACHE [%1+#-64], D0Ar6\n" \
451 "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \
452 "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \
453 "GETL D0.5, D1.5, [A0StP+#-24]\n" \
454 "GETL D0.6, D1.6, [A0StP+#-16]\n" \
455 "GETL D0.7, D1.7, [A0StP+#-8]\n" \
456 "SUB A0StP, A0StP, #40\n" \
457 " .section .fixup,\"ax\"\n" \
458 "4:\n" \
459 " ADD %0, %0, #4\n" \
460 "3:\n" \
461 " MOV D0Ar2, TXSTATUS\n" \
462 " MOV D1Ar1, TXSTATUS\n" \
463 " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \
464 " MOV TXSTATUS, D1Ar1\n" \
465 FIXUP \
466 " MOVT D0Ar2,#HI(1b)\n" \
467 " JUMP D0Ar2,#LO(1b)\n" \
468 " .previous\n" \
469 " .section __ex_table,\"a\"\n" \
470 " .long 21b,3b\n" \
471 " .long 22b,3b\n" \
472 " .long 23b,3b\n" \
473 " .long 24b,3b\n" \
474 " .long 25b,3b\n" \
475 " .long 26b,3b\n" \
476 " .long 27b,3b\n" \
477 " .long 28b,3b\n" \
478 " .long 29b,3b\n" \
479 " .long 30b,3b\n" \
480 " .long 31b,3b\n" \
481 " .long 32b,3b\n" \
482 " .long 33b,3b\n" \
483 " .long 34b,3b\n" \
484 " .long 35b,3b\n" \
485 " .long 36b,3b\n" \
486 " .long 37b,4b\n" \
487 " .previous\n" \
488 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
489 : "0" (to), "1" (from), "2" (ret), "3" (n) \
490 : "D1Ar1", "D0Ar2", "memory")
491
492 /* rewind 'to' and 'from' pointers when a fault occurs
493 *
494 * Rationale:
495 * A fault always occurs on writing to user buffer. A fault
496 * is at a single address, so we need to rewind by only 4
497 * bytes.
498 * Since we do a complete read from kernel buffer before
499 * writing, we need to rewind it also. The amount to be
500 * rewind equals the number of faulty writes in MSETD
501 * which is: [4 - (LSM_STEP-1)]*4
502 * LSM_STEP is bits 10:8 in TXSTATUS which is already read
503 * and stored in D0Ar2
504 *
505 * NOTE: If a fault occurs at the last operation in M{G,S}ETL
506 * LSM_STEP will be 0. ie: we do 4 writes in our case, if
507 * a fault happens at the 4th write, LSM_STEP will be 0
508 * instead of 4. The code copes with that.
509 *
510 * n is updated by the number of successful writes, which is:
511 * n = n - (LSM_STEP-1)*4
512 */
513 #define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
514 __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
515 "LSR D0Ar2, D0Ar2, #8\n" \
516 "AND D0Ar2, D0Ar2, #0x7\n" \
517 "ADDZ D0Ar2, D0Ar2, #4\n" \
518 "SUB D0Ar2, D0Ar2, #1\n" \
519 "MOV D1Ar1, #4\n" \
520 "SUB D0Ar2, D1Ar1, D0Ar2\n" \
521 "LSL D0Ar2, D0Ar2, #2\n" \
522 "LSL D1Ar1, D1Ar1, #2\n" \
523 "SUB D1Ar1, D1Ar1, D0Ar2\n" \
524 "SUB %0, %0, #4\n" \
525 "SUB %1, %1, D0Ar2\n" \
526 "SUB %3, %3, D1Ar1\n")
527
__copy_user(void __user * pdst,const void * psrc,unsigned long n)528 unsigned long __copy_user(void __user *pdst, const void *psrc,
529 unsigned long n)
530 {
531 register char __user *dst asm ("A0.2") = pdst;
532 register const char *src asm ("A1.2") = psrc;
533 unsigned long retn = 0;
534
535 if (n == 0)
536 return 0;
537
538 if ((unsigned long) src & 1) {
539 __asm_copy_to_user_1(dst, src, retn);
540 n--;
541 }
542 if ((unsigned long) dst & 1) {
543 /* Worst case - byte copy */
544 while (n > 0) {
545 __asm_copy_to_user_1(dst, src, retn);
546 n--;
547 }
548 }
549 if (((unsigned long) src & 2) && n >= 2) {
550 __asm_copy_to_user_2(dst, src, retn);
551 n -= 2;
552 }
553 if ((unsigned long) dst & 2) {
554 /* Second worst case - word copy */
555 while (n >= 2) {
556 __asm_copy_to_user_2(dst, src, retn);
557 n -= 2;
558 }
559 }
560
561 #ifdef USE_RAPF
562 /* 64 bit copy loop */
563 if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
564 if (n >= RAPF_MIN_BUF_SIZE) {
565 /* copy user using 64 bit rapf copy */
566 __asm_copy_to_user_64bit_rapf_loop(dst, src, retn,
567 n, "64cu");
568 }
569 while (n >= 8) {
570 __asm_copy_to_user_8x64(dst, src, retn);
571 n -= 8;
572 }
573 }
574 if (n >= RAPF_MIN_BUF_SIZE) {
575 /* copy user using 32 bit rapf copy */
576 __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu");
577 }
578 #else
579 /* 64 bit copy loop */
580 if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
581 while (n >= 8) {
582 __asm_copy_to_user_8x64(dst, src, retn);
583 n -= 8;
584 }
585 }
586 #endif
587
588 while (n >= 16) {
589 __asm_copy_to_user_16(dst, src, retn);
590 n -= 16;
591 }
592
593 while (n >= 4) {
594 __asm_copy_to_user_4(dst, src, retn);
595 n -= 4;
596 }
597
598 switch (n) {
599 case 0:
600 break;
601 case 1:
602 __asm_copy_to_user_1(dst, src, retn);
603 break;
604 case 2:
605 __asm_copy_to_user_2(dst, src, retn);
606 break;
607 case 3:
608 __asm_copy_to_user_3(dst, src, retn);
609 break;
610 }
611
612 return retn;
613 }
614 EXPORT_SYMBOL(__copy_user);
615
616 #define __asm_copy_from_user_1(to, from, ret) \
617 __asm_copy_user_cont(to, from, ret, \
618 " GETB D1Ar1,[%1++]\n" \
619 "2: SETB [%0++],D1Ar1\n", \
620 "3: ADD %2,%2,#1\n" \
621 " SETB [%0++],D1Ar1\n", \
622 " .long 2b,3b\n")
623
624 #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
625 __asm_copy_user_cont(to, from, ret, \
626 " GETW D1Ar1,[%1++]\n" \
627 "2: SETW [%0++],D1Ar1\n" COPY, \
628 "3: ADD %2,%2,#2\n" \
629 " SETW [%0++],D1Ar1\n" FIXUP, \
630 " .long 2b,3b\n" TENTRY)
631
632 #define __asm_copy_from_user_2(to, from, ret) \
633 __asm_copy_from_user_2x_cont(to, from, ret, "", "", "")
634
635 #define __asm_copy_from_user_3(to, from, ret) \
636 __asm_copy_from_user_2x_cont(to, from, ret, \
637 " GETB D1Ar1,[%1++]\n" \
638 "4: SETB [%0++],D1Ar1\n", \
639 "5: ADD %2,%2,#1\n" \
640 " SETB [%0++],D1Ar1\n", \
641 " .long 4b,5b\n")
642
643 #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
644 __asm_copy_user_cont(to, from, ret, \
645 " GETD D1Ar1,[%1++]\n" \
646 "2: SETD [%0++],D1Ar1\n" COPY, \
647 "3: ADD %2,%2,#4\n" \
648 " SETD [%0++],D1Ar1\n" FIXUP, \
649 " .long 2b,3b\n" TENTRY)
650
651 #define __asm_copy_from_user_4(to, from, ret) \
652 __asm_copy_from_user_4x_cont(to, from, ret, "", "", "")
653
654 #define __asm_copy_from_user_5(to, from, ret) \
655 __asm_copy_from_user_4x_cont(to, from, ret, \
656 " GETB D1Ar1,[%1++]\n" \
657 "4: SETB [%0++],D1Ar1\n", \
658 "5: ADD %2,%2,#1\n" \
659 " SETB [%0++],D1Ar1\n", \
660 " .long 4b,5b\n")
661
662 #define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
663 __asm_copy_from_user_4x_cont(to, from, ret, \
664 " GETW D1Ar1,[%1++]\n" \
665 "4: SETW [%0++],D1Ar1\n" COPY, \
666 "5: ADD %2,%2,#2\n" \
667 " SETW [%0++],D1Ar1\n" FIXUP, \
668 " .long 4b,5b\n" TENTRY)
669
670 #define __asm_copy_from_user_6(to, from, ret) \
671 __asm_copy_from_user_6x_cont(to, from, ret, "", "", "")
672
673 #define __asm_copy_from_user_7(to, from, ret) \
674 __asm_copy_from_user_6x_cont(to, from, ret, \
675 " GETB D1Ar1,[%1++]\n" \
676 "6: SETB [%0++],D1Ar1\n", \
677 "7: ADD %2,%2,#1\n" \
678 " SETB [%0++],D1Ar1\n", \
679 " .long 6b,7b\n")
680
681 #define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
682 __asm_copy_from_user_4x_cont(to, from, ret, \
683 " GETD D1Ar1,[%1++]\n" \
684 "4: SETD [%0++],D1Ar1\n" COPY, \
685 "5: ADD %2,%2,#4\n" \
686 " SETD [%0++],D1Ar1\n" FIXUP, \
687 " .long 4b,5b\n" TENTRY)
688
689 #define __asm_copy_from_user_8(to, from, ret) \
690 __asm_copy_from_user_8x_cont(to, from, ret, "", "", "")
691
692 #define __asm_copy_from_user_9(to, from, ret) \
693 __asm_copy_from_user_8x_cont(to, from, ret, \
694 " GETB D1Ar1,[%1++]\n" \
695 "6: SETB [%0++],D1Ar1\n", \
696 "7: ADD %2,%2,#1\n" \
697 " SETB [%0++],D1Ar1\n", \
698 " .long 6b,7b\n")
699
700 #define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
701 __asm_copy_from_user_8x_cont(to, from, ret, \
702 " GETW D1Ar1,[%1++]\n" \
703 "6: SETW [%0++],D1Ar1\n" COPY, \
704 "7: ADD %2,%2,#2\n" \
705 " SETW [%0++],D1Ar1\n" FIXUP, \
706 " .long 6b,7b\n" TENTRY)
707
708 #define __asm_copy_from_user_10(to, from, ret) \
709 __asm_copy_from_user_10x_cont(to, from, ret, "", "", "")
710
711 #define __asm_copy_from_user_11(to, from, ret) \
712 __asm_copy_from_user_10x_cont(to, from, ret, \
713 " GETB D1Ar1,[%1++]\n" \
714 "8: SETB [%0++],D1Ar1\n", \
715 "9: ADD %2,%2,#1\n" \
716 " SETB [%0++],D1Ar1\n", \
717 " .long 8b,9b\n")
718
719 #define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
720 __asm_copy_from_user_8x_cont(to, from, ret, \
721 " GETD D1Ar1,[%1++]\n" \
722 "6: SETD [%0++],D1Ar1\n" COPY, \
723 "7: ADD %2,%2,#4\n" \
724 " SETD [%0++],D1Ar1\n" FIXUP, \
725 " .long 6b,7b\n" TENTRY)
726
727 #define __asm_copy_from_user_12(to, from, ret) \
728 __asm_copy_from_user_12x_cont(to, from, ret, "", "", "")
729
730 #define __asm_copy_from_user_13(to, from, ret) \
731 __asm_copy_from_user_12x_cont(to, from, ret, \
732 " GETB D1Ar1,[%1++]\n" \
733 "8: SETB [%0++],D1Ar1\n", \
734 "9: ADD %2,%2,#1\n" \
735 " SETB [%0++],D1Ar1\n", \
736 " .long 8b,9b\n")
737
738 #define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
739 __asm_copy_from_user_12x_cont(to, from, ret, \
740 " GETW D1Ar1,[%1++]\n" \
741 "8: SETW [%0++],D1Ar1\n" COPY, \
742 "9: ADD %2,%2,#2\n" \
743 " SETW [%0++],D1Ar1\n" FIXUP, \
744 " .long 8b,9b\n" TENTRY)
745
746 #define __asm_copy_from_user_14(to, from, ret) \
747 __asm_copy_from_user_14x_cont(to, from, ret, "", "", "")
748
749 #define __asm_copy_from_user_15(to, from, ret) \
750 __asm_copy_from_user_14x_cont(to, from, ret, \
751 " GETB D1Ar1,[%1++]\n" \
752 "10: SETB [%0++],D1Ar1\n", \
753 "11: ADD %2,%2,#1\n" \
754 " SETB [%0++],D1Ar1\n", \
755 " .long 10b,11b\n")
756
757 #define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
758 __asm_copy_from_user_12x_cont(to, from, ret, \
759 " GETD D1Ar1,[%1++]\n" \
760 "8: SETD [%0++],D1Ar1\n" COPY, \
761 "9: ADD %2,%2,#4\n" \
762 " SETD [%0++],D1Ar1\n" FIXUP, \
763 " .long 8b,9b\n" TENTRY)
764
765 #define __asm_copy_from_user_16(to, from, ret) \
766 __asm_copy_from_user_16x_cont(to, from, ret, "", "", "")
767
768 #define __asm_copy_from_user_8x64(to, from, ret) \
769 asm volatile ( \
770 " GETL D0Ar2,D1Ar1,[%1++]\n" \
771 "2: SETL [%0++],D0Ar2,D1Ar1\n" \
772 "1:\n" \
773 " .section .fixup,\"ax\"\n" \
774 " MOV D1Ar1,#0\n" \
775 " MOV D0Ar2,#0\n" \
776 "3: ADD %2,%2,#8\n" \
777 " SETL [%0++],D0Ar2,D1Ar1\n" \
778 " MOVT D0Ar2,#HI(1b)\n" \
779 " JUMP D0Ar2,#LO(1b)\n" \
780 " .previous\n" \
781 " .section __ex_table,\"a\"\n" \
782 " .long 2b,3b\n" \
783 " .previous\n" \
784 : "=a" (to), "=r" (from), "=r" (ret) \
785 : "0" (to), "1" (from), "2" (ret) \
786 : "D1Ar1", "D0Ar2", "memory")
787
788 /* rewind 'from' pointer when a fault occurs
789 *
790 * Rationale:
791 * A fault occurs while reading from user buffer, which is the
792 * source. Since the fault is at a single address, we only
793 * need to rewind by 8 bytes.
794 * Since we don't write to kernel buffer until we read first,
795 * the kernel buffer is at the right state and needn't be
796 * corrected.
797 */
798 #define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \
799 __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
800 "SUB %1, %1, #8\n")
801
802 /* rewind 'from' pointer when a fault occurs
803 *
804 * Rationale:
805 * A fault occurs while reading from user buffer, which is the
806 * source. Since the fault is at a single address, we only
807 * need to rewind by 4 bytes.
808 * Since we don't write to kernel buffer until we read first,
809 * the kernel buffer is at the right state and needn't be
810 * corrected.
811 */
812 #define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \
813 __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
814 "SUB %1, %1, #4\n")
815
816
817 /* Copy from user to kernel, zeroing the bytes that were inaccessible in
818 userland. The return-value is the number of bytes that were
819 inaccessible. */
__copy_user_zeroing(void * pdst,const void __user * psrc,unsigned long n)820 unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
821 unsigned long n)
822 {
823 register char *dst asm ("A0.2") = pdst;
824 register const char __user *src asm ("A1.2") = psrc;
825 unsigned long retn = 0;
826
827 if (n == 0)
828 return 0;
829
830 if ((unsigned long) src & 1) {
831 __asm_copy_from_user_1(dst, src, retn);
832 n--;
833 }
834 if ((unsigned long) dst & 1) {
835 /* Worst case - byte copy */
836 while (n > 0) {
837 __asm_copy_from_user_1(dst, src, retn);
838 n--;
839 if (retn)
840 goto copy_exception_bytes;
841 }
842 }
843 if (((unsigned long) src & 2) && n >= 2) {
844 __asm_copy_from_user_2(dst, src, retn);
845 n -= 2;
846 }
847 if ((unsigned long) dst & 2) {
848 /* Second worst case - word copy */
849 while (n >= 2) {
850 __asm_copy_from_user_2(dst, src, retn);
851 n -= 2;
852 if (retn)
853 goto copy_exception_bytes;
854 }
855 }
856
857 /* We only need one check after the unalignment-adjustments,
858 because if both adjustments were done, either both or
859 neither reference had an exception. */
860 if (retn != 0)
861 goto copy_exception_bytes;
862
863 #ifdef USE_RAPF
864 /* 64 bit copy loop */
865 if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
866 if (n >= RAPF_MIN_BUF_SIZE) {
867 /* Copy using fast 64bit rapf */
868 __asm_copy_from_user_64bit_rapf_loop(dst, src, retn,
869 n, "64cuz");
870 }
871 while (n >= 8) {
872 __asm_copy_from_user_8x64(dst, src, retn);
873 n -= 8;
874 if (retn)
875 goto copy_exception_bytes;
876 }
877 }
878
879 if (n >= RAPF_MIN_BUF_SIZE) {
880 /* Copy using fast 32bit rapf */
881 __asm_copy_from_user_32bit_rapf_loop(dst, src, retn,
882 n, "32cuz");
883 }
884 #else
885 /* 64 bit copy loop */
886 if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
887 while (n >= 8) {
888 __asm_copy_from_user_8x64(dst, src, retn);
889 n -= 8;
890 if (retn)
891 goto copy_exception_bytes;
892 }
893 }
894 #endif
895
896 while (n >= 4) {
897 __asm_copy_from_user_4(dst, src, retn);
898 n -= 4;
899
900 if (retn)
901 goto copy_exception_bytes;
902 }
903
904 /* If we get here, there were no memory read faults. */
905 switch (n) {
906 /* These copies are at least "naturally aligned" (so we don't
907 have to check each byte), due to the src alignment code.
908 The *_3 case *will* get the correct count for retn. */
909 case 0:
910 /* This case deliberately left in (if you have doubts check the
911 generated assembly code). */
912 break;
913 case 1:
914 __asm_copy_from_user_1(dst, src, retn);
915 break;
916 case 2:
917 __asm_copy_from_user_2(dst, src, retn);
918 break;
919 case 3:
920 __asm_copy_from_user_3(dst, src, retn);
921 break;
922 }
923
924 /* If we get here, retn correctly reflects the number of failing
925 bytes. */
926 return retn;
927
928 copy_exception_bytes:
929 /* We already have "retn" bytes cleared, and need to clear the
930 remaining "n" bytes. A non-optimized simple byte-for-byte in-line
931 memset is preferred here, since this isn't speed-critical code and
932 we'd rather have this a leaf-function than calling memset. */
933 {
934 char *endp;
935 for (endp = dst + n; dst < endp; dst++)
936 *dst = 0;
937 }
938
939 return retn + n;
940 }
941 EXPORT_SYMBOL(__copy_user_zeroing);
942
943 #define __asm_clear_8x64(to, ret) \
944 asm volatile ( \
945 " MOV D0Ar2,#0\n" \
946 " MOV D1Ar1,#0\n" \
947 " SETL [%0],D0Ar2,D1Ar1\n" \
948 "2: SETL [%0++],D0Ar2,D1Ar1\n" \
949 "1:\n" \
950 " .section .fixup,\"ax\"\n" \
951 "3: ADD %1,%1,#8\n" \
952 " MOVT D0Ar2,#HI(1b)\n" \
953 " JUMP D0Ar2,#LO(1b)\n" \
954 " .previous\n" \
955 " .section __ex_table,\"a\"\n" \
956 " .long 2b,3b\n" \
957 " .previous\n" \
958 : "=r" (to), "=r" (ret) \
959 : "0" (to), "1" (ret) \
960 : "D1Ar1", "D0Ar2", "memory")
961
962 /* Zero userspace. */
963
964 #define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \
965 asm volatile ( \
966 " MOV D1Ar1,#0\n" \
967 CLEAR \
968 "1:\n" \
969 " .section .fixup,\"ax\"\n" \
970 FIXUP \
971 " MOVT D1Ar1,#HI(1b)\n" \
972 " JUMP D1Ar1,#LO(1b)\n" \
973 " .previous\n" \
974 " .section __ex_table,\"a\"\n" \
975 TENTRY \
976 " .previous" \
977 : "=r" (to), "=r" (ret) \
978 : "0" (to), "1" (ret) \
979 : "D1Ar1", "memory")
980
981 #define __asm_clear_1(to, ret) \
982 __asm_clear(to, ret, \
983 " SETB [%0],D1Ar1\n" \
984 "2: SETB [%0++],D1Ar1\n", \
985 "3: ADD %1,%1,#1\n", \
986 " .long 2b,3b\n")
987
988 #define __asm_clear_2(to, ret) \
989 __asm_clear(to, ret, \
990 " SETW [%0],D1Ar1\n" \
991 "2: SETW [%0++],D1Ar1\n", \
992 "3: ADD %1,%1,#2\n", \
993 " .long 2b,3b\n")
994
995 #define __asm_clear_3(to, ret) \
996 __asm_clear(to, ret, \
997 "2: SETW [%0++],D1Ar1\n" \
998 " SETB [%0],D1Ar1\n" \
999 "3: SETB [%0++],D1Ar1\n", \
1000 "4: ADD %1,%1,#2\n" \
1001 "5: ADD %1,%1,#1\n", \
1002 " .long 2b,4b\n" \
1003 " .long 3b,5b\n")
1004
1005 #define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1006 __asm_clear(to, ret, \
1007 " SETD [%0],D1Ar1\n" \
1008 "2: SETD [%0++],D1Ar1\n" CLEAR, \
1009 "3: ADD %1,%1,#4\n" FIXUP, \
1010 " .long 2b,3b\n" TENTRY)
1011
1012 #define __asm_clear_4(to, ret) \
1013 __asm_clear_4x_cont(to, ret, "", "", "")
1014
1015 #define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1016 __asm_clear_4x_cont(to, ret, \
1017 " SETD [%0],D1Ar1\n" \
1018 "4: SETD [%0++],D1Ar1\n" CLEAR, \
1019 "5: ADD %1,%1,#4\n" FIXUP, \
1020 " .long 4b,5b\n" TENTRY)
1021
1022 #define __asm_clear_8(to, ret) \
1023 __asm_clear_8x_cont(to, ret, "", "", "")
1024
1025 #define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1026 __asm_clear_8x_cont(to, ret, \
1027 " SETD [%0],D1Ar1\n" \
1028 "6: SETD [%0++],D1Ar1\n" CLEAR, \
1029 "7: ADD %1,%1,#4\n" FIXUP, \
1030 " .long 6b,7b\n" TENTRY)
1031
1032 #define __asm_clear_12(to, ret) \
1033 __asm_clear_12x_cont(to, ret, "", "", "")
1034
1035 #define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
1036 __asm_clear_12x_cont(to, ret, \
1037 " SETD [%0],D1Ar1\n" \
1038 "8: SETD [%0++],D1Ar1\n" CLEAR, \
1039 "9: ADD %1,%1,#4\n" FIXUP, \
1040 " .long 8b,9b\n" TENTRY)
1041
1042 #define __asm_clear_16(to, ret) \
1043 __asm_clear_16x_cont(to, ret, "", "", "")
1044
__do_clear_user(void __user * pto,unsigned long pn)1045 unsigned long __do_clear_user(void __user *pto, unsigned long pn)
1046 {
1047 register char __user *dst asm ("D0Re0") = pto;
1048 register unsigned long n asm ("D1Re0") = pn;
1049 register unsigned long retn asm ("D0Ar6") = 0;
1050
1051 if ((unsigned long) dst & 1) {
1052 __asm_clear_1(dst, retn);
1053 n--;
1054 }
1055
1056 if ((unsigned long) dst & 2) {
1057 __asm_clear_2(dst, retn);
1058 n -= 2;
1059 }
1060
1061 /* 64 bit copy loop */
1062 if (!((__force unsigned long) dst & 7)) {
1063 while (n >= 8) {
1064 __asm_clear_8x64(dst, retn);
1065 n -= 8;
1066 }
1067 }
1068
1069 while (n >= 16) {
1070 __asm_clear_16(dst, retn);
1071 n -= 16;
1072 }
1073
1074 while (n >= 4) {
1075 __asm_clear_4(dst, retn);
1076 n -= 4;
1077 }
1078
1079 switch (n) {
1080 case 0:
1081 break;
1082 case 1:
1083 __asm_clear_1(dst, retn);
1084 break;
1085 case 2:
1086 __asm_clear_2(dst, retn);
1087 break;
1088 case 3:
1089 __asm_clear_3(dst, retn);
1090 break;
1091 }
1092
1093 return retn;
1094 }
1095 EXPORT_SYMBOL(__do_clear_user);
1096
__get_user_asm_b(const void __user * addr,long * err)1097 unsigned char __get_user_asm_b(const void __user *addr, long *err)
1098 {
1099 register unsigned char x asm ("D0Re0") = 0;
1100 asm volatile (
1101 " GETB %0,[%2]\n"
1102 "1:\n"
1103 " GETB %0,[%2]\n"
1104 "2:\n"
1105 " .section .fixup,\"ax\"\n"
1106 "3: MOV D0FrT,%3\n"
1107 " SETD [%1],D0FrT\n"
1108 " MOVT D0FrT,#HI(2b)\n"
1109 " JUMP D0FrT,#LO(2b)\n"
1110 " .previous\n"
1111 " .section __ex_table,\"a\"\n"
1112 " .long 1b,3b\n"
1113 " .previous\n"
1114 : "=r" (x)
1115 : "r" (err), "r" (addr), "P" (-EFAULT)
1116 : "D0FrT");
1117 return x;
1118 }
1119 EXPORT_SYMBOL(__get_user_asm_b);
1120
__get_user_asm_w(const void __user * addr,long * err)1121 unsigned short __get_user_asm_w(const void __user *addr, long *err)
1122 {
1123 register unsigned short x asm ("D0Re0") = 0;
1124 asm volatile (
1125 " GETW %0,[%2]\n"
1126 "1:\n"
1127 " GETW %0,[%2]\n"
1128 "2:\n"
1129 " .section .fixup,\"ax\"\n"
1130 "3: MOV D0FrT,%3\n"
1131 " SETD [%1],D0FrT\n"
1132 " MOVT D0FrT,#HI(2b)\n"
1133 " JUMP D0FrT,#LO(2b)\n"
1134 " .previous\n"
1135 " .section __ex_table,\"a\"\n"
1136 " .long 1b,3b\n"
1137 " .previous\n"
1138 : "=r" (x)
1139 : "r" (err), "r" (addr), "P" (-EFAULT)
1140 : "D0FrT");
1141 return x;
1142 }
1143 EXPORT_SYMBOL(__get_user_asm_w);
1144
__get_user_asm_d(const void __user * addr,long * err)1145 unsigned int __get_user_asm_d(const void __user *addr, long *err)
1146 {
1147 register unsigned int x asm ("D0Re0") = 0;
1148 asm volatile (
1149 " GETD %0,[%2]\n"
1150 "1:\n"
1151 " GETD %0,[%2]\n"
1152 "2:\n"
1153 " .section .fixup,\"ax\"\n"
1154 "3: MOV D0FrT,%3\n"
1155 " SETD [%1],D0FrT\n"
1156 " MOVT D0FrT,#HI(2b)\n"
1157 " JUMP D0FrT,#LO(2b)\n"
1158 " .previous\n"
1159 " .section __ex_table,\"a\"\n"
1160 " .long 1b,3b\n"
1161 " .previous\n"
1162 : "=r" (x)
1163 : "r" (err), "r" (addr), "P" (-EFAULT)
1164 : "D0FrT");
1165 return x;
1166 }
1167 EXPORT_SYMBOL(__get_user_asm_d);
1168
__put_user_asm_b(unsigned int x,void __user * addr)1169 long __put_user_asm_b(unsigned int x, void __user *addr)
1170 {
1171 register unsigned int err asm ("D0Re0") = 0;
1172 asm volatile (
1173 " MOV %0,#0\n"
1174 " SETB [%2],%1\n"
1175 "1:\n"
1176 " SETB [%2],%1\n"
1177 "2:\n"
1178 ".section .fixup,\"ax\"\n"
1179 "3: MOV %0,%3\n"
1180 " MOVT D0FrT,#HI(2b)\n"
1181 " JUMP D0FrT,#LO(2b)\n"
1182 ".previous\n"
1183 ".section __ex_table,\"a\"\n"
1184 " .long 1b,3b\n"
1185 ".previous"
1186 : "=r"(err)
1187 : "d" (x), "a" (addr), "P"(-EFAULT)
1188 : "D0FrT");
1189 return err;
1190 }
1191 EXPORT_SYMBOL(__put_user_asm_b);
1192
__put_user_asm_w(unsigned int x,void __user * addr)1193 long __put_user_asm_w(unsigned int x, void __user *addr)
1194 {
1195 register unsigned int err asm ("D0Re0") = 0;
1196 asm volatile (
1197 " MOV %0,#0\n"
1198 " SETW [%2],%1\n"
1199 "1:\n"
1200 " SETW [%2],%1\n"
1201 "2:\n"
1202 ".section .fixup,\"ax\"\n"
1203 "3: MOV %0,%3\n"
1204 " MOVT D0FrT,#HI(2b)\n"
1205 " JUMP D0FrT,#LO(2b)\n"
1206 ".previous\n"
1207 ".section __ex_table,\"a\"\n"
1208 " .long 1b,3b\n"
1209 ".previous"
1210 : "=r"(err)
1211 : "d" (x), "a" (addr), "P"(-EFAULT)
1212 : "D0FrT");
1213 return err;
1214 }
1215 EXPORT_SYMBOL(__put_user_asm_w);
1216
__put_user_asm_d(unsigned int x,void __user * addr)1217 long __put_user_asm_d(unsigned int x, void __user *addr)
1218 {
1219 register unsigned int err asm ("D0Re0") = 0;
1220 asm volatile (
1221 " MOV %0,#0\n"
1222 " SETD [%2],%1\n"
1223 "1:\n"
1224 " SETD [%2],%1\n"
1225 "2:\n"
1226 ".section .fixup,\"ax\"\n"
1227 "3: MOV %0,%3\n"
1228 " MOVT D0FrT,#HI(2b)\n"
1229 " JUMP D0FrT,#LO(2b)\n"
1230 ".previous\n"
1231 ".section __ex_table,\"a\"\n"
1232 " .long 1b,3b\n"
1233 ".previous"
1234 : "=r"(err)
1235 : "d" (x), "a" (addr), "P"(-EFAULT)
1236 : "D0FrT");
1237 return err;
1238 }
1239 EXPORT_SYMBOL(__put_user_asm_d);
1240
__put_user_asm_l(unsigned long long x,void __user * addr)1241 long __put_user_asm_l(unsigned long long x, void __user *addr)
1242 {
1243 register unsigned int err asm ("D0Re0") = 0;
1244 asm volatile (
1245 " MOV %0,#0\n"
1246 " SETL [%2],%1,%t1\n"
1247 "1:\n"
1248 " SETL [%2],%1,%t1\n"
1249 "2:\n"
1250 ".section .fixup,\"ax\"\n"
1251 "3: MOV %0,%3\n"
1252 " MOVT D0FrT,#HI(2b)\n"
1253 " JUMP D0FrT,#LO(2b)\n"
1254 ".previous\n"
1255 ".section __ex_table,\"a\"\n"
1256 " .long 1b,3b\n"
1257 ".previous"
1258 : "=r"(err)
1259 : "d" (x), "a" (addr), "P"(-EFAULT)
1260 : "D0FrT");
1261 return err;
1262 }
1263 EXPORT_SYMBOL(__put_user_asm_l);
1264
strnlen_user(const char __user * src,long count)1265 long strnlen_user(const char __user *src, long count)
1266 {
1267 long res;
1268
1269 if (!access_ok(VERIFY_READ, src, 0))
1270 return 0;
1271
1272 asm volatile (" MOV D0Ar4, %1\n"
1273 " MOV D0Ar6, %2\n"
1274 "0:\n"
1275 " SUBS D0FrT, D0Ar6, #0\n"
1276 " SUB D0Ar6, D0Ar6, #1\n"
1277 " BLE 2f\n"
1278 " GETB D0FrT, [D0Ar4+#1++]\n"
1279 "1:\n"
1280 " TST D0FrT, #255\n"
1281 " BNE 0b\n"
1282 "2:\n"
1283 " SUB %0, %2, D0Ar6\n"
1284 "3:\n"
1285 " .section .fixup,\"ax\"\n"
1286 "4:\n"
1287 " MOV %0, #0\n"
1288 " MOVT D0FrT,#HI(3b)\n"
1289 " JUMP D0FrT,#LO(3b)\n"
1290 " .previous\n"
1291 " .section __ex_table,\"a\"\n"
1292 " .long 1b,4b\n"
1293 " .previous\n"
1294 : "=r" (res)
1295 : "r" (src), "r" (count)
1296 : "D0FrT", "D0Ar4", "D0Ar6", "cc");
1297
1298 return res;
1299 }
1300 EXPORT_SYMBOL(strnlen_user);
1301
__strncpy_from_user(char * dst,const char __user * src,long count)1302 long __strncpy_from_user(char *dst, const char __user *src, long count)
1303 {
1304 long res;
1305
1306 if (count == 0)
1307 return 0;
1308
1309 /*
1310 * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop.
1311 * So do we.
1312 *
1313 * This code is deduced from:
1314 *
1315 * char tmp2;
1316 * long tmp1, tmp3;
1317 * tmp1 = count;
1318 * while ((*dst++ = (tmp2 = *src++)) != 0
1319 * && --tmp1)
1320 * ;
1321 *
1322 * res = count - tmp1;
1323 *
1324 * with tweaks.
1325 */
1326
1327 asm volatile (" MOV %0,%3\n"
1328 "1:\n"
1329 " GETB D0FrT,[%2++]\n"
1330 "2:\n"
1331 " CMP D0FrT,#0\n"
1332 " SETB [%1++],D0FrT\n"
1333 " BEQ 3f\n"
1334 " SUBS %0,%0,#1\n"
1335 " BNZ 1b\n"
1336 "3:\n"
1337 " SUB %0,%3,%0\n"
1338 "4:\n"
1339 " .section .fixup,\"ax\"\n"
1340 "5:\n"
1341 " MOV %0,%7\n"
1342 " MOVT D0FrT,#HI(4b)\n"
1343 " JUMP D0FrT,#LO(4b)\n"
1344 " .previous\n"
1345 " .section __ex_table,\"a\"\n"
1346 " .long 2b,5b\n"
1347 " .previous"
1348 : "=r" (res), "=r" (dst), "=r" (src), "=r" (count)
1349 : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT)
1350 : "D0FrT", "memory", "cc");
1351
1352 return res;
1353 }
1354 EXPORT_SYMBOL(__strncpy_from_user);
1355