1/* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * vineetg: June 2010 9 * -__clear_user( ) called multiple times during elf load was byte loop 10 * converted to do as much word clear as possible. 11 * 12 * vineetg: Dec 2009 13 * -Hand crafted constant propagation for "constant" copy sizes 14 * -stock kernel shrunk by 33K at -O3 15 * 16 * vineetg: Sept 2009 17 * -Added option to (UN)inline copy_(to|from)_user to reduce code sz 18 * -kernel shrunk by 200K even at -O3 (gcc 4.2.1) 19 * -Enabled when doing -Os 20 * 21 * Amit Bhor, Sameer Dhavale: Codito Technologies 2004 22 */ 23 24#ifndef _ASM_ARC_UACCESS_H 25#define _ASM_ARC_UACCESS_H 26 27#include <linux/sched.h> 28#include <asm/errno.h> 29#include <linux/string.h> /* for generic string functions */ 30 31 32#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) 33 34/* 35 * Algorthmically, for __user_ok() we want do: 36 * (start < TASK_SIZE) && (start+len < TASK_SIZE) 37 * where TASK_SIZE could either be retrieved from thread_info->addr_limit or 38 * emitted directly in code. 39 * 40 * This can however be rewritten as follows: 41 * (len <= TASK_SIZE) && (start+len < TASK_SIZE) 42 * 43 * Because it essentially checks if buffer end is within limit and @len is 44 * non-ngeative, which implies that buffer start will be within limit too. 45 * 46 * The reason for rewriting being, for majority of cases, @len is generally 47 * compile time constant, causing first sub-expression to be compile time 48 * subsumed. 49 * 50 * The second part would generate weird large LIMMs e.g. (0x6000_0000 - 0x10), 51 * so we check for TASK_SIZE using get_fs() since the addr_limit load from mem 52 * would already have been done at this call site for __kernel_ok() 53 * 54 */ 55#define __user_ok(addr, sz) (((sz) <= TASK_SIZE) && \ 56 ((addr) <= (get_fs() - (sz)))) 57#define __access_ok(addr, sz) (unlikely(__kernel_ok) || \ 58 likely(__user_ok((addr), (sz)))) 59 60/*********** Single byte/hword/word copies ******************/ 61 62#define __get_user_fn(sz, u, k) \ 63({ \ 64 long __ret = 0; /* success by default */ \ 65 switch (sz) { \ 66 case 1: __arc_get_user_one(*(k), u, "ldb", __ret); break; \ 67 case 2: __arc_get_user_one(*(k), u, "ldw", __ret); break; \ 68 case 4: __arc_get_user_one(*(k), u, "ld", __ret); break; \ 69 case 8: __arc_get_user_one_64(*(k), u, __ret); break; \ 70 } \ 71 __ret; \ 72}) 73 74/* 75 * Returns 0 on success, -EFAULT if not. 76 * @ret already contains 0 - given that errors will be less likely 77 * (hence +r asm constraint below). 78 * In case of error, fixup code will make it -EFAULT 79 */ 80#define __arc_get_user_one(dst, src, op, ret) \ 81 __asm__ __volatile__( \ 82 "1: "op" %1,[%2]\n" \ 83 "2: ;nop\n" \ 84 " .section .fixup, \"ax\"\n" \ 85 " .align 4\n" \ 86 "3: mov %0, %3\n" \ 87 " j 2b\n" \ 88 " .previous\n" \ 89 " .section __ex_table, \"a\"\n" \ 90 " .align 4\n" \ 91 " .word 1b,3b\n" \ 92 " .previous\n" \ 93 \ 94 : "+r" (ret), "=r" (dst) \ 95 : "r" (src), "ir" (-EFAULT)) 96 97#define __arc_get_user_one_64(dst, src, ret) \ 98 __asm__ __volatile__( \ 99 "1: ld %1,[%2]\n" \ 100 "4: ld %R1,[%2, 4]\n" \ 101 "2: ;nop\n" \ 102 " .section .fixup, \"ax\"\n" \ 103 " .align 4\n" \ 104 "3: mov %0, %3\n" \ 105 " j 2b\n" \ 106 " .previous\n" \ 107 " .section __ex_table, \"a\"\n" \ 108 " .align 4\n" \ 109 " .word 1b,3b\n" \ 110 " .word 4b,3b\n" \ 111 " .previous\n" \ 112 \ 113 : "+r" (ret), "=r" (dst) \ 114 : "r" (src), "ir" (-EFAULT)) 115 116#define __put_user_fn(sz, u, k) \ 117({ \ 118 long __ret = 0; /* success by default */ \ 119 switch (sz) { \ 120 case 1: __arc_put_user_one(*(k), u, "stb", __ret); break; \ 121 case 2: __arc_put_user_one(*(k), u, "stw", __ret); break; \ 122 case 4: __arc_put_user_one(*(k), u, "st", __ret); break; \ 123 case 8: __arc_put_user_one_64(*(k), u, __ret); break; \ 124 } \ 125 __ret; \ 126}) 127 128#define __arc_put_user_one(src, dst, op, ret) \ 129 __asm__ __volatile__( \ 130 "1: "op" %1,[%2]\n" \ 131 "2: ;nop\n" \ 132 " .section .fixup, \"ax\"\n" \ 133 " .align 4\n" \ 134 "3: mov %0, %3\n" \ 135 " j 2b\n" \ 136 " .previous\n" \ 137 " .section __ex_table, \"a\"\n" \ 138 " .align 4\n" \ 139 " .word 1b,3b\n" \ 140 " .previous\n" \ 141 \ 142 : "+r" (ret) \ 143 : "r" (src), "r" (dst), "ir" (-EFAULT)) 144 145#define __arc_put_user_one_64(src, dst, ret) \ 146 __asm__ __volatile__( \ 147 "1: st %1,[%2]\n" \ 148 "4: st %R1,[%2, 4]\n" \ 149 "2: ;nop\n" \ 150 " .section .fixup, \"ax\"\n" \ 151 " .align 4\n" \ 152 "3: mov %0, %3\n" \ 153 " j 2b\n" \ 154 " .previous\n" \ 155 " .section __ex_table, \"a\"\n" \ 156 " .align 4\n" \ 157 " .word 1b,3b\n" \ 158 " .word 4b,3b\n" \ 159 " .previous\n" \ 160 \ 161 : "+r" (ret) \ 162 : "r" (src), "r" (dst), "ir" (-EFAULT)) 163 164 165static inline unsigned long 166__arc_copy_from_user(void *to, const void __user *from, unsigned long n) 167{ 168 long res = 0; 169 char val; 170 unsigned long tmp1, tmp2, tmp3, tmp4; 171 unsigned long orig_n = n; 172 173 if (n == 0) 174 return 0; 175 176 /* unaligned */ 177 if (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3)) { 178 179 unsigned char tmp; 180 181 __asm__ __volatile__ ( 182 " mov.f lp_count, %0 \n" 183 " lpnz 2f \n" 184 "1: ldb.ab %1, [%3, 1] \n" 185 " stb.ab %1, [%2, 1] \n" 186 " sub %0,%0,1 \n" 187 "2: ;nop \n" 188 " .section .fixup, \"ax\" \n" 189 " .align 4 \n" 190 "3: j 2b \n" 191 " .previous \n" 192 " .section __ex_table, \"a\" \n" 193 " .align 4 \n" 194 " .word 1b, 3b \n" 195 " .previous \n" 196 197 : "+r" (n), 198 /* 199 * Note as an '&' earlyclobber operand to make sure the 200 * temporary register inside the loop is not the same as 201 * FROM or TO. 202 */ 203 "=&r" (tmp), "+r" (to), "+r" (from) 204 : 205 : "lp_count", "lp_start", "lp_end", "memory"); 206 207 return n; 208 } 209 210 /* 211 * Hand-crafted constant propagation to reduce code sz of the 212 * laddered copy 16x,8,4,2,1 213 */ 214 if (__builtin_constant_p(orig_n)) { 215 res = orig_n; 216 217 if (orig_n / 16) { 218 orig_n = orig_n % 16; 219 220 __asm__ __volatile__( 221 " lsr lp_count, %7,4 \n" 222 " lp 3f \n" 223 "1: ld.ab %3, [%2, 4] \n" 224 "11: ld.ab %4, [%2, 4] \n" 225 "12: ld.ab %5, [%2, 4] \n" 226 "13: ld.ab %6, [%2, 4] \n" 227 " st.ab %3, [%1, 4] \n" 228 " st.ab %4, [%1, 4] \n" 229 " st.ab %5, [%1, 4] \n" 230 " st.ab %6, [%1, 4] \n" 231 " sub %0,%0,16 \n" 232 "3: ;nop \n" 233 " .section .fixup, \"ax\" \n" 234 " .align 4 \n" 235 "4: j 3b \n" 236 " .previous \n" 237 " .section __ex_table, \"a\" \n" 238 " .align 4 \n" 239 " .word 1b, 4b \n" 240 " .word 11b,4b \n" 241 " .word 12b,4b \n" 242 " .word 13b,4b \n" 243 " .previous \n" 244 : "+r" (res), "+r"(to), "+r"(from), 245 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4) 246 : "ir"(n) 247 : "lp_count", "memory"); 248 } 249 if (orig_n / 8) { 250 orig_n = orig_n % 8; 251 252 __asm__ __volatile__( 253 "14: ld.ab %3, [%2,4] \n" 254 "15: ld.ab %4, [%2,4] \n" 255 " st.ab %3, [%1,4] \n" 256 " st.ab %4, [%1,4] \n" 257 " sub %0,%0,8 \n" 258 "31: ;nop \n" 259 " .section .fixup, \"ax\" \n" 260 " .align 4 \n" 261 "4: j 31b \n" 262 " .previous \n" 263 " .section __ex_table, \"a\" \n" 264 " .align 4 \n" 265 " .word 14b,4b \n" 266 " .word 15b,4b \n" 267 " .previous \n" 268 : "+r" (res), "+r"(to), "+r"(from), 269 "=r"(tmp1), "=r"(tmp2) 270 : 271 : "memory"); 272 } 273 if (orig_n / 4) { 274 orig_n = orig_n % 4; 275 276 __asm__ __volatile__( 277 "16: ld.ab %3, [%2,4] \n" 278 " st.ab %3, [%1,4] \n" 279 " sub %0,%0,4 \n" 280 "32: ;nop \n" 281 " .section .fixup, \"ax\" \n" 282 " .align 4 \n" 283 "4: j 32b \n" 284 " .previous \n" 285 " .section __ex_table, \"a\" \n" 286 " .align 4 \n" 287 " .word 16b,4b \n" 288 " .previous \n" 289 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1) 290 : 291 : "memory"); 292 } 293 if (orig_n / 2) { 294 orig_n = orig_n % 2; 295 296 __asm__ __volatile__( 297 "17: ldw.ab %3, [%2,2] \n" 298 " stw.ab %3, [%1,2] \n" 299 " sub %0,%0,2 \n" 300 "33: ;nop \n" 301 " .section .fixup, \"ax\" \n" 302 " .align 4 \n" 303 "4: j 33b \n" 304 " .previous \n" 305 " .section __ex_table, \"a\" \n" 306 " .align 4 \n" 307 " .word 17b,4b \n" 308 " .previous \n" 309 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1) 310 : 311 : "memory"); 312 } 313 if (orig_n & 1) { 314 __asm__ __volatile__( 315 "18: ldb.ab %3, [%2,2] \n" 316 " stb.ab %3, [%1,2] \n" 317 " sub %0,%0,1 \n" 318 "34: ; nop \n" 319 " .section .fixup, \"ax\" \n" 320 " .align 4 \n" 321 "4: j 34b \n" 322 " .previous \n" 323 " .section __ex_table, \"a\" \n" 324 " .align 4 \n" 325 " .word 18b,4b \n" 326 " .previous \n" 327 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1) 328 : 329 : "memory"); 330 } 331 } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */ 332 333 __asm__ __volatile__( 334 " mov %0,%3 \n" 335 " lsr.f lp_count, %3,4 \n" /* 16x bytes */ 336 " lpnz 3f \n" 337 "1: ld.ab %5, [%2, 4] \n" 338 "11: ld.ab %6, [%2, 4] \n" 339 "12: ld.ab %7, [%2, 4] \n" 340 "13: ld.ab %8, [%2, 4] \n" 341 " st.ab %5, [%1, 4] \n" 342 " st.ab %6, [%1, 4] \n" 343 " st.ab %7, [%1, 4] \n" 344 " st.ab %8, [%1, 4] \n" 345 " sub %0,%0,16 \n" 346 "3: and.f %3,%3,0xf \n" /* stragglers */ 347 " bz 34f \n" 348 " bbit0 %3,3,31f \n" /* 8 bytes left */ 349 "14: ld.ab %5, [%2,4] \n" 350 "15: ld.ab %6, [%2,4] \n" 351 " st.ab %5, [%1,4] \n" 352 " st.ab %6, [%1,4] \n" 353 " sub.f %0,%0,8 \n" 354 "31: bbit0 %3,2,32f \n" /* 4 bytes left */ 355 "16: ld.ab %5, [%2,4] \n" 356 " st.ab %5, [%1,4] \n" 357 " sub.f %0,%0,4 \n" 358 "32: bbit0 %3,1,33f \n" /* 2 bytes left */ 359 "17: ldw.ab %5, [%2,2] \n" 360 " stw.ab %5, [%1,2] \n" 361 " sub.f %0,%0,2 \n" 362 "33: bbit0 %3,0,34f \n" 363 "18: ldb.ab %5, [%2,1] \n" /* 1 byte left */ 364 " stb.ab %5, [%1,1] \n" 365 " sub.f %0,%0,1 \n" 366 "34: ;nop \n" 367 " .section .fixup, \"ax\" \n" 368 " .align 4 \n" 369 "4: j 34b \n" 370 " .previous \n" 371 " .section __ex_table, \"a\" \n" 372 " .align 4 \n" 373 " .word 1b, 4b \n" 374 " .word 11b,4b \n" 375 " .word 12b,4b \n" 376 " .word 13b,4b \n" 377 " .word 14b,4b \n" 378 " .word 15b,4b \n" 379 " .word 16b,4b \n" 380 " .word 17b,4b \n" 381 " .word 18b,4b \n" 382 " .previous \n" 383 : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val), 384 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4) 385 : 386 : "lp_count", "memory"); 387 } 388 389 return res; 390} 391 392extern unsigned long slowpath_copy_to_user(void __user *to, const void *from, 393 unsigned long n); 394 395static inline unsigned long 396__arc_copy_to_user(void __user *to, const void *from, unsigned long n) 397{ 398 long res = 0; 399 char val; 400 unsigned long tmp1, tmp2, tmp3, tmp4; 401 unsigned long orig_n = n; 402 403 if (n == 0) 404 return 0; 405 406 /* unaligned */ 407 if (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3)) { 408 409 unsigned char tmp; 410 411 __asm__ __volatile__( 412 " mov.f lp_count, %0 \n" 413 " lpnz 3f \n" 414 " ldb.ab %1, [%3, 1] \n" 415 "1: stb.ab %1, [%2, 1] \n" 416 " sub %0, %0, 1 \n" 417 "3: ;nop \n" 418 " .section .fixup, \"ax\" \n" 419 " .align 4 \n" 420 "4: j 3b \n" 421 " .previous \n" 422 " .section __ex_table, \"a\" \n" 423 " .align 4 \n" 424 " .word 1b, 4b \n" 425 " .previous \n" 426 427 : "+r" (n), 428 /* Note as an '&' earlyclobber operand to make sure the 429 * temporary register inside the loop is not the same as 430 * FROM or TO. 431 */ 432 "=&r" (tmp), "+r" (to), "+r" (from) 433 : 434 : "lp_count", "lp_start", "lp_end", "memory"); 435 436 return n; 437 } 438 439 if (__builtin_constant_p(orig_n)) { 440 res = orig_n; 441 442 if (orig_n / 16) { 443 orig_n = orig_n % 16; 444 445 __asm__ __volatile__( 446 " lsr lp_count, %7,4 \n" 447 " lp 3f \n" 448 " ld.ab %3, [%2, 4] \n" 449 " ld.ab %4, [%2, 4] \n" 450 " ld.ab %5, [%2, 4] \n" 451 " ld.ab %6, [%2, 4] \n" 452 "1: st.ab %3, [%1, 4] \n" 453 "11: st.ab %4, [%1, 4] \n" 454 "12: st.ab %5, [%1, 4] \n" 455 "13: st.ab %6, [%1, 4] \n" 456 " sub %0, %0, 16 \n" 457 "3:;nop \n" 458 " .section .fixup, \"ax\" \n" 459 " .align 4 \n" 460 "4: j 3b \n" 461 " .previous \n" 462 " .section __ex_table, \"a\" \n" 463 " .align 4 \n" 464 " .word 1b, 4b \n" 465 " .word 11b,4b \n" 466 " .word 12b,4b \n" 467 " .word 13b,4b \n" 468 " .previous \n" 469 : "+r" (res), "+r"(to), "+r"(from), 470 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4) 471 : "ir"(n) 472 : "lp_count", "memory"); 473 } 474 if (orig_n / 8) { 475 orig_n = orig_n % 8; 476 477 __asm__ __volatile__( 478 " ld.ab %3, [%2,4] \n" 479 " ld.ab %4, [%2,4] \n" 480 "14: st.ab %3, [%1,4] \n" 481 "15: st.ab %4, [%1,4] \n" 482 " sub %0, %0, 8 \n" 483 "31:;nop \n" 484 " .section .fixup, \"ax\" \n" 485 " .align 4 \n" 486 "4: j 31b \n" 487 " .previous \n" 488 " .section __ex_table, \"a\" \n" 489 " .align 4 \n" 490 " .word 14b,4b \n" 491 " .word 15b,4b \n" 492 " .previous \n" 493 : "+r" (res), "+r"(to), "+r"(from), 494 "=r"(tmp1), "=r"(tmp2) 495 : 496 : "memory"); 497 } 498 if (orig_n / 4) { 499 orig_n = orig_n % 4; 500 501 __asm__ __volatile__( 502 " ld.ab %3, [%2,4] \n" 503 "16: st.ab %3, [%1,4] \n" 504 " sub %0, %0, 4 \n" 505 "32:;nop \n" 506 " .section .fixup, \"ax\" \n" 507 " .align 4 \n" 508 "4: j 32b \n" 509 " .previous \n" 510 " .section __ex_table, \"a\" \n" 511 " .align 4 \n" 512 " .word 16b,4b \n" 513 " .previous \n" 514 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1) 515 : 516 : "memory"); 517 } 518 if (orig_n / 2) { 519 orig_n = orig_n % 2; 520 521 __asm__ __volatile__( 522 " ldw.ab %3, [%2,2] \n" 523 "17: stw.ab %3, [%1,2] \n" 524 " sub %0, %0, 2 \n" 525 "33:;nop \n" 526 " .section .fixup, \"ax\" \n" 527 " .align 4 \n" 528 "4: j 33b \n" 529 " .previous \n" 530 " .section __ex_table, \"a\" \n" 531 " .align 4 \n" 532 " .word 17b,4b \n" 533 " .previous \n" 534 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1) 535 : 536 : "memory"); 537 } 538 if (orig_n & 1) { 539 __asm__ __volatile__( 540 " ldb.ab %3, [%2,1] \n" 541 "18: stb.ab %3, [%1,1] \n" 542 " sub %0, %0, 1 \n" 543 "34: ;nop \n" 544 " .section .fixup, \"ax\" \n" 545 " .align 4 \n" 546 "4: j 34b \n" 547 " .previous \n" 548 " .section __ex_table, \"a\" \n" 549 " .align 4 \n" 550 " .word 18b,4b \n" 551 " .previous \n" 552 : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1) 553 : 554 : "memory"); 555 } 556 } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */ 557 558 __asm__ __volatile__( 559 " mov %0,%3 \n" 560 " lsr.f lp_count, %3,4 \n" /* 16x bytes */ 561 " lpnz 3f \n" 562 " ld.ab %5, [%2, 4] \n" 563 " ld.ab %6, [%2, 4] \n" 564 " ld.ab %7, [%2, 4] \n" 565 " ld.ab %8, [%2, 4] \n" 566 "1: st.ab %5, [%1, 4] \n" 567 "11: st.ab %6, [%1, 4] \n" 568 "12: st.ab %7, [%1, 4] \n" 569 "13: st.ab %8, [%1, 4] \n" 570 " sub %0, %0, 16 \n" 571 "3: and.f %3,%3,0xf \n" /* stragglers */ 572 " bz 34f \n" 573 " bbit0 %3,3,31f \n" /* 8 bytes left */ 574 " ld.ab %5, [%2,4] \n" 575 " ld.ab %6, [%2,4] \n" 576 "14: st.ab %5, [%1,4] \n" 577 "15: st.ab %6, [%1,4] \n" 578 " sub.f %0, %0, 8 \n" 579 "31: bbit0 %3,2,32f \n" /* 4 bytes left */ 580 " ld.ab %5, [%2,4] \n" 581 "16: st.ab %5, [%1,4] \n" 582 " sub.f %0, %0, 4 \n" 583 "32: bbit0 %3,1,33f \n" /* 2 bytes left */ 584 " ldw.ab %5, [%2,2] \n" 585 "17: stw.ab %5, [%1,2] \n" 586 " sub.f %0, %0, 2 \n" 587 "33: bbit0 %3,0,34f \n" 588 " ldb.ab %5, [%2,1] \n" /* 1 byte left */ 589 "18: stb.ab %5, [%1,1] \n" 590 " sub.f %0, %0, 1 \n" 591 "34: ;nop \n" 592 " .section .fixup, \"ax\" \n" 593 " .align 4 \n" 594 "4: j 34b \n" 595 " .previous \n" 596 " .section __ex_table, \"a\" \n" 597 " .align 4 \n" 598 " .word 1b, 4b \n" 599 " .word 11b,4b \n" 600 " .word 12b,4b \n" 601 " .word 13b,4b \n" 602 " .word 14b,4b \n" 603 " .word 15b,4b \n" 604 " .word 16b,4b \n" 605 " .word 17b,4b \n" 606 " .word 18b,4b \n" 607 " .previous \n" 608 : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val), 609 "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4) 610 : 611 : "lp_count", "memory"); 612 } 613 614 return res; 615} 616 617static inline unsigned long __arc_clear_user(void __user *to, unsigned long n) 618{ 619 long res = n; 620 unsigned char *d_char = to; 621 622 __asm__ __volatile__( 623 " bbit0 %0, 0, 1f \n" 624 "75: stb.ab %2, [%0,1] \n" 625 " sub %1, %1, 1 \n" 626 "1: bbit0 %0, 1, 2f \n" 627 "76: stw.ab %2, [%0,2] \n" 628 " sub %1, %1, 2 \n" 629 "2: asr.f lp_count, %1, 2 \n" 630 " lpnz 3f \n" 631 "77: st.ab %2, [%0,4] \n" 632 " sub %1, %1, 4 \n" 633 "3: bbit0 %1, 1, 4f \n" 634 "78: stw.ab %2, [%0,2] \n" 635 " sub %1, %1, 2 \n" 636 "4: bbit0 %1, 0, 5f \n" 637 "79: stb.ab %2, [%0,1] \n" 638 " sub %1, %1, 1 \n" 639 "5: \n" 640 " .section .fixup, \"ax\" \n" 641 " .align 4 \n" 642 "3: j 5b \n" 643 " .previous \n" 644 " .section __ex_table, \"a\" \n" 645 " .align 4 \n" 646 " .word 75b, 3b \n" 647 " .word 76b, 3b \n" 648 " .word 77b, 3b \n" 649 " .word 78b, 3b \n" 650 " .word 79b, 3b \n" 651 " .previous \n" 652 : "+r"(d_char), "+r"(res) 653 : "i"(0) 654 : "lp_count", "lp_start", "lp_end", "memory"); 655 656 return res; 657} 658 659static inline long 660__arc_strncpy_from_user(char *dst, const char __user *src, long count) 661{ 662 long res = count; 663 char val; 664 unsigned int hw_count; 665 666 if (count == 0) 667 return 0; 668 669 __asm__ __volatile__( 670 " lp 2f \n" 671 "1: ldb.ab %3, [%2, 1] \n" 672 " breq.d %3, 0, 2f \n" 673 " stb.ab %3, [%1, 1] \n" 674 "2: sub %0, %6, %4 \n" 675 "3: ;nop \n" 676 " .section .fixup, \"ax\" \n" 677 " .align 4 \n" 678 "4: mov %0, %5 \n" 679 " j 3b \n" 680 " .previous \n" 681 " .section __ex_table, \"a\" \n" 682 " .align 4 \n" 683 " .word 1b, 4b \n" 684 " .previous \n" 685 : "=r"(res), "+r"(dst), "+r"(src), "=&r"(val), "=l"(hw_count) 686 : "g"(-EFAULT), "ir"(count), "4"(count) /* this "4" seeds lp_count */ 687 : "memory"); 688 689 return res; 690} 691 692static inline long __arc_strnlen_user(const char __user *s, long n) 693{ 694 long res, tmp1, cnt; 695 char val; 696 697 __asm__ __volatile__( 698 " mov %2, %1 \n" 699 "1: ldb.ab %3, [%0, 1] \n" 700 " breq.d %3, 0, 2f \n" 701 " sub.f %2, %2, 1 \n" 702 " bnz 1b \n" 703 " sub %2, %2, 1 \n" 704 "2: sub %0, %1, %2 \n" 705 "3: ;nop \n" 706 " .section .fixup, \"ax\" \n" 707 " .align 4 \n" 708 "4: mov %0, 0 \n" 709 " j 3b \n" 710 " .previous \n" 711 " .section __ex_table, \"a\" \n" 712 " .align 4 \n" 713 " .word 1b, 4b \n" 714 " .previous \n" 715 : "=r"(res), "=r"(tmp1), "=r"(cnt), "=r"(val) 716 : "0"(s), "1"(n) 717 : "memory"); 718 719 return res; 720} 721 722#ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE 723#define __copy_from_user(t, f, n) __arc_copy_from_user(t, f, n) 724#define __copy_to_user(t, f, n) __arc_copy_to_user(t, f, n) 725#define __clear_user(d, n) __arc_clear_user(d, n) 726#define __strncpy_from_user(d, s, n) __arc_strncpy_from_user(d, s, n) 727#define __strnlen_user(s, n) __arc_strnlen_user(s, n) 728#else 729extern long arc_copy_from_user_noinline(void *to, const void __user * from, 730 unsigned long n); 731extern long arc_copy_to_user_noinline(void __user *to, const void *from, 732 unsigned long n); 733extern unsigned long arc_clear_user_noinline(void __user *to, 734 unsigned long n); 735extern long arc_strncpy_from_user_noinline (char *dst, const char __user *src, 736 long count); 737extern long arc_strnlen_user_noinline(const char __user *src, long n); 738 739#define __copy_from_user(t, f, n) arc_copy_from_user_noinline(t, f, n) 740#define __copy_to_user(t, f, n) arc_copy_to_user_noinline(t, f, n) 741#define __clear_user(d, n) arc_clear_user_noinline(d, n) 742#define __strncpy_from_user(d, s, n) arc_strncpy_from_user_noinline(d, s, n) 743#define __strnlen_user(s, n) arc_strnlen_user_noinline(s, n) 744 745#endif 746 747#include <asm-generic/uaccess.h> 748 749extern int fixup_exception(struct pt_regs *regs); 750 751#endif 752