[PATCH] s390: speedup strn{cpy,len}_from_user.

From: Martin Schwidefsky <schwidefsky@de.ibm.com>

From: Martin Schwidefsky <schwidefsky@de.ibm.com>

Speedup strncpy_from_user and strnlen_from_user by using the search string
instruction in the secondary space mode.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/s390/lib/uaccess.S b/arch/s390/lib/uaccess.S
index ef26df4c9579aa667203bc909f32e96dcd6dff08..e8029ef42ef2d3c13f679d50626c6aaf0608508b 100644 (file)
--- a/arch/s390/lib/uaccess.S
+++ b/arch/s390/lib/uaccess.S
@@ -154,46 +154,57 @@ __clear_user_asm:
         .align 4
         .text
         .globl __strncpy_from_user_asm
-       # %r2 = dst, %r3 = src, %r4 = count
+       # %r2 = count, %r3 = dst, %r4 = src
 __strncpy_from_user_asm:
        lhi     %r0,0
-       lhi     %r1,1
-       lhi     %r5,0
-0:     mvcp    0(%r1,%r2),0(%r3),%r0
-       tm      0(%r2),0xff
-       jz      1f
-       la      %r2,1(%r2)
-       la      %r3,1(%r3)
-       ahi     %r5,1
-       clr     %r5,%r4
-       jl      0b
-1:     lr      %r2,%r5
+       lr      %r1,%r4
+       la      %r4,0(%r4)      # clear high order bit from %r4
+       la      %r2,0(%r2,%r4)  # %r2 points to first byte after string
+       sacf    256
+0:     srst    %r2,%r1
+       jo      0b
+       sacf    0
+       lr      %r1,%r2
+       jh      1f              # \0 found in string ?
+       ahi     %r1,1           # include \0 in copy
+1:     slr     %r1,%r4         # %r1 = copy length (without \0)
+       slr     %r2,%r4         # %r2 = return length (including \0)
+2:     mvcp    0(%r1,%r3),0(%r4),%r0
+       jnz     3f
        br      %r14
-2:     lhi     %r2,-EFAULT
+3:     la      %r3,256(%r3)
+       la      %r4,256(%r4)
+       ahi     %r1,-256
+       mvcp    0(%r1,%r3),0(%r4),%r0
+       jnz     3b
        br      %r14
-        .section __ex_table,"a"
-       .long   0b,2b
+4:     sacf    0
+       lhi     %r2,-EFAULT
+       br      %r14
+       .section __ex_table,"a"
+       .long   0b,4b
        .previous
 
         .align 4
         .text
         .globl __strnlen_user_asm
-       # %r2 = src, %r3 = count
+       # %r2 = count, %r3 = src
 __strnlen_user_asm:
        lhi     %r0,0
-       lhi     %r1,1
-       lhi     %r5,0
-0:     mvcp    24(%r1,%r15),0(%r2),%r0
-       ahi     %r5,1
-       tm      24(%r15),0xff
-       jz      1f
-       la      %r2,1(%r2)
-       clr     %r5,%r3
-       jl      0b
-1:     lr      %r2,%r5
+       lr      %r1,%r3
+       la      %r3,0(%r3)      # clear high order bit from %r4
+       la      %r2,0(%r2,%r3)  # %r2 points to first byte after string
+       sacf    256
+0:     srst    %r2,%r1
+       jo      0b
+       sacf    0
+       jh      1f              # \0 found in string ?
+       ahi     %r2,1           # strnlen_user result includes the \0
+1:     slr     %r2,%r3
        br      %r14
-2:     lhi     %r2,-EFAULT
+2:     sacf    0
+       lhi     %r2,-EFAULT
        br      %r14
-        .section __ex_table,"a"
+       .section __ex_table,"a"
        .long   0b,2b
        .previous

diff --git a/arch/s390/lib/uaccess64.S b/arch/s390/lib/uaccess64.S
index c12f70a67b9780dcfd665fb3bdbfcca1f16f69b5..0ca56972f4f090501b46608fb0d6d41b0bfa732e 100644 (file)
--- a/arch/s390/lib/uaccess64.S
+++ b/arch/s390/lib/uaccess64.S
@@ -152,46 +152,55 @@ __clear_user_asm:
         .align 4
         .text
         .globl __strncpy_from_user_asm
-       # %r2 = dst, %r3 = src, %r4 = count
+       # %r2 = count, %r3 = dst, %r4 = src
 __strncpy_from_user_asm:
        lghi    %r0,0
-       lghi    %r1,1
-       lghi    %r5,0
-0:     mvcp    0(%r1,%r2),0(%r3),%r0
-       tm      0(%r2),0xff
-       jz      1f
-       la      %r2,1(%r2)
-       la      %r3,1(%r3)
-       aghi    %r5,1
-       clgr    %r5,%r4
-       jl      0b
-1:     lgr     %r2,%r5
+       lgr     %r1,%r4
+       la      %r2,0(%r2,%r4)  # %r2 points to first byte after string
+       sacf    256
+0:     srst    %r2,%r1
+       jo      0b
+       sacf    0
+       lgr     %r1,%r2
+       jh      1f              # \0 found in string ?
+       aghi    %r1,1           # include \0 in copy
+1:     slgr    %r1,%r4         # %r1 = copy length (without \0)
+       slgr    %r2,%r4         # %r2 = return length (including \0)
+2:     mvcp    0(%r1,%r3),0(%r4),%r0
+       jnz     3f
        br      %r14
-2:     lghi    %r2,-EFAULT
+3:     la      %r3,256(%r3)
+       la      %r4,256(%r4)
+       aghi    %r1,-256
+       mvcp    0(%r1,%r3),0(%r4),%r0
+       jnz     3b
        br      %r14
-        .section __ex_table,"a"
-       .quad   0b,2b
+4:     sacf    0
+       lghi    %r2,-EFAULT
+       br      %r14
+       .section __ex_table,"a"
+       .quad   0b,4b
        .previous
 
         .align 4
         .text
         .globl __strnlen_user_asm
-       # %r2 = src, %r3 = count
+       # %r2 = count, %r3 = src
 __strnlen_user_asm:
        lghi    %r0,0
-       lghi    %r1,1
-       lghi    %r5,0
-0:     mvcp    24(%r1,%r15),0(%r2),%r0
-       aghi    %r5,1
-       tm      24(%r15),0xff
-       jz      1f
-       la      %r2,1(%r2)
-       clgr    %r5,%r3
-       jl      0b
-1:     lgr     %r2,%r5
+       lgr     %r1,%r3
+       la      %r2,0(%r2,%r3)  # %r2 points to first byte after string
+       sacf    256
+0:     srst    %r2,%r1
+       jo      0b
+       sacf    0
+       jh      1f              # \0 found in string ?
+       aghi    %r2,1           # strnlen_user result includes the \0
+1:     slgr    %r2,%r3
        br      %r14
-2:     lghi    %r2,-EFAULT
+2:     sacf    0
+       lghi    %r2,-EFAULT
        br      %r14
-        .section __ex_table,"a"
+       .section __ex_table,"a"
        .quad   0b,2b
        .previous

diff --git a/include/asm-s390/uaccess.h b/include/asm-s390/uaccess.h
index f520c12c7d6c1662e7e46522c942d38ca16f72dd..bec57b7507b9fd251af88a7da6e8800962302247 100644 (file)
--- a/include/asm-s390/uaccess.h
+++ b/include/asm-s390/uaccess.h
@@ -367,8 +367,8 @@ copy_in_user(void __user *to, const void __user *from, unsigned long n)
 /*
  * Copy a null terminated string from userspace.
  */
-extern long __strncpy_from_user_asm(char *dst, const char __user *src,
-                                                               long count);
+extern long __strncpy_from_user_asm(long count, char *dst,
+                                       const char __user *src);
 
 static inline long
 strncpy_from_user(char *dst, const char __user *src, long count)
@@ -376,18 +376,18 @@ strncpy_from_user(char *dst, const char __user *src, long count)
         long res = -EFAULT;
         might_sleep();
         if (access_ok(VERIFY_READ, src, 1))
-                res = __strncpy_from_user_asm(dst, src, count);
+                res = __strncpy_from_user_asm(count, dst, src);
         return res;
 }
 
 
-extern long __strnlen_user_asm(const char __user *src, long count);
+extern long __strnlen_user_asm(long count, const char __user *src);
 
 static inline unsigned long
 strnlen_user(const char __user * src, unsigned long n)
 {
        might_sleep();
-       return __strnlen_user_asm(src, n);
+       return __strnlen_user_asm(n, src);
 }
 
 /**