1/* 2 * arch/s390/lib/uaccess_mvcos.c 3 * 4 * Optimized user space space access functions based on mvcos. 5 * 6 * Copyright (C) IBM Corp. 2006 7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), 8 * Gerald Schaefer (gerald.schaefer@de.ibm.com) 9 */ 10 11#include <linux/errno.h> 12#include <linux/mm.h> 13#include <asm/uaccess.h> 14#include <asm/futex.h> 15#include "uaccess.h" 16 17#ifndef __s390x__ 18#define AHI "ahi" 19#define ALR "alr" 20#define CLR "clr" 21#define LHI "lhi" 22#define SLR "slr" 23#else 24#define AHI "aghi" 25#define ALR "algr" 26#define CLR "clgr" 27#define LHI "lghi" 28#define SLR "slgr" 29#endif 30 31static size_t copy_from_user_mvcos(size_t size, const void __user *ptr, void *x) 32{ 33 register unsigned long reg0 asm("0") = 0x81UL; 34 unsigned long tmp1, tmp2; 35 36 tmp1 = -4096UL; 37 asm volatile( 38 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n" 39 "9: jz 7f\n" 40 "1:"ALR" %0,%3\n" 41 " "SLR" %1,%3\n" 42 " "SLR" %2,%3\n" 43 " j 0b\n" 44 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 45 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 46 " "SLR" %4,%1\n" 47 " "CLR" %0,%4\n" /* copy crosses next page boundary? */ 48 " jnh 4f\n" 49 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n" 50 "10:"SLR" %0,%4\n" 51 " "ALR" %2,%4\n" 52 "4:"LHI" %4,-1\n" 53 " "ALR" %4,%0\n" /* copy remaining size, subtract 1 */ 54 " bras %3,6f\n" /* memset loop */ 55 " xc 0(1,%2),0(%2)\n" 56 "5: xc 0(256,%2),0(%2)\n" 57 " la %2,256(%2)\n" 58 "6:"AHI" %4,-256\n" 59 " jnm 5b\n" 60 " ex %4,0(%3)\n" 61 " j 8f\n" 62 "7:"SLR" %0,%0\n" 63 "8: \n" 64 EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b) 65 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 66 : "d" (reg0) : "cc", "memory"); 67 return size; 68} 69 70static size_t copy_from_user_mvcos_check(size_t size, const void __user *ptr, void *x) 71{ 72 if (size <= 256) 73 return copy_from_user_std(size, ptr, x); 74 return copy_from_user_mvcos(size, ptr, x); 75} 76 77static size_t copy_to_user_mvcos(size_t size, void __user *ptr, const void *x) 78{ 79 register unsigned long reg0 asm("0") = 0x810000UL; 80 unsigned long tmp1, tmp2; 81 82 tmp1 = -4096UL; 83 asm volatile( 84 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 85 "6: jz 4f\n" 86 "1:"ALR" %0,%3\n" 87 " "SLR" %1,%3\n" 88 " "SLR" %2,%3\n" 89 " j 0b\n" 90 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 91 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 92 " "SLR" %4,%1\n" 93 " "CLR" %0,%4\n" /* copy crosses next page boundary? */ 94 " jnh 5f\n" 95 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n" 96 "7:"SLR" %0,%4\n" 97 " j 5f\n" 98 "4:"SLR" %0,%0\n" 99 "5: \n" 100 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) 101 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 102 : "d" (reg0) : "cc", "memory"); 103 return size; 104} 105 106static size_t copy_to_user_mvcos_check(size_t size, void __user *ptr, 107 const void *x) 108{ 109 if (size <= 256) 110 return copy_to_user_std(size, ptr, x); 111 return copy_to_user_mvcos(size, ptr, x); 112} 113 114static size_t copy_in_user_mvcos(size_t size, void __user *to, 115 const void __user *from) 116{ 117 register unsigned long reg0 asm("0") = 0x810081UL; 118 unsigned long tmp1, tmp2; 119 120 tmp1 = -4096UL; 121 asm volatile( 122 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 123 " jz 2f\n" 124 "1:"ALR" %0,%3\n" 125 " "SLR" %1,%3\n" 126 " "SLR" %2,%3\n" 127 " j 0b\n" 128 "2:"SLR" %0,%0\n" 129 "3: \n" 130 EX_TABLE(0b,3b) 131 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2) 132 : "d" (reg0) : "cc", "memory"); 133 return size; 134} 135 136static size_t clear_user_mvcos(size_t size, void __user *to) 137{ 138 register unsigned long reg0 asm("0") = 0x810000UL; 139 unsigned long tmp1, tmp2; 140 141 tmp1 = -4096UL; 142 asm volatile( 143 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n" 144 " jz 4f\n" 145 "1:"ALR" %0,%2\n" 146 " "SLR" %1,%2\n" 147 " j 0b\n" 148 "2: la %3,4095(%1)\n"/* %4 = to + 4095 */ 149 " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */ 150 " "SLR" %3,%1\n" 151 " "CLR" %0,%3\n" /* copy crosses next page boundary? */ 152 " jnh 5f\n" 153 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n" 154 " "SLR" %0,%3\n" 155 " j 5f\n" 156 "4:"SLR" %0,%0\n" 157 "5: \n" 158 EX_TABLE(0b,2b) EX_TABLE(3b,5b) 159 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) 160 : "a" (empty_zero_page), "d" (reg0) : "cc", "memory"); 161 return size; 162} 163 164static size_t strnlen_user_mvcos(size_t count, const char __user *src) 165{ 166 char buf[256]; 167 int rc; 168 size_t done, len, len_str; 169 170 done = 0; 171 do { 172 len = min(count - done, (size_t) 256); 173 rc = uaccess.copy_from_user(len, src + done, buf); 174 if (unlikely(rc == len)) 175 return 0; 176 len -= rc; 177 len_str = strnlen(buf, len); 178 done += len_str; 179 } while ((len_str == len) && (done < count)); 180 return done + 1; 181} 182 183static size_t strncpy_from_user_mvcos(size_t count, const char __user *src, 184 char *dst) 185{ 186 int rc; 187 size_t done, len, len_str; 188 189 done = 0; 190 do { 191 len = min(count - done, (size_t) 4096); 192 rc = uaccess.copy_from_user(len, src + done, dst); 193 if (unlikely(rc == len)) 194 return -EFAULT; 195 len -= rc; 196 len_str = strnlen(dst, len); 197 done += len_str; 198 } while ((len_str == len) && (done < count)); 199 return done; 200} 201 202struct uaccess_ops uaccess_mvcos = { 203 .copy_from_user = copy_from_user_mvcos_check, 204 .copy_from_user_small = copy_from_user_std, 205 .copy_to_user = copy_to_user_mvcos_check, 206 .copy_to_user_small = copy_to_user_std, 207 .copy_in_user = copy_in_user_mvcos, 208 .clear_user = clear_user_mvcos, 209 .strnlen_user = strnlen_user_std, 210 .strncpy_from_user = strncpy_from_user_std, 211 .futex_atomic_op = futex_atomic_op_std, 212 .futex_atomic_cmpxchg = futex_atomic_cmpxchg_std, 213}; 214 215struct uaccess_ops uaccess_mvcos_switch = { 216 .copy_from_user = copy_from_user_mvcos, 217 .copy_from_user_small = copy_from_user_mvcos, 218 .copy_to_user = copy_to_user_mvcos, 219 .copy_to_user_small = copy_to_user_mvcos, 220 .copy_in_user = copy_in_user_mvcos, 221 .clear_user = clear_user_mvcos, 222 .strnlen_user = strnlen_user_mvcos, 223 .strncpy_from_user = strncpy_from_user_mvcos, 224 .futex_atomic_op = futex_atomic_op_pt, 225 .futex_atomic_cmpxchg = futex_atomic_cmpxchg_pt, 226}; 227