1/* 2 * atomic64_t for 586+ 3 * 4 * Copyright �� 2010 Luca Barbieri 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 */ 11 12#include <linux/linkage.h> 13#include <asm/alternative-asm.h> 14#include <asm/dwarf2.h> 15 16.macro SAVE reg 17 pushl %\reg 18 CFI_ADJUST_CFA_OFFSET 4 19 CFI_REL_OFFSET \reg, 0 20.endm 21 22.macro RESTORE reg 23 popl %\reg 24 CFI_ADJUST_CFA_OFFSET -4 25 CFI_RESTORE \reg 26.endm 27 28.macro read64 reg 29 movl %ebx, %eax 30 movl %ecx, %edx 31/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 32 LOCK_PREFIX 33 cmpxchg8b (\reg) 34.endm 35 36ENTRY(atomic64_read_cx8) 37 CFI_STARTPROC 38 39 read64 %ecx 40 ret 41 CFI_ENDPROC 42ENDPROC(atomic64_read_cx8) 43 44ENTRY(atomic64_set_cx8) 45 CFI_STARTPROC 46 471: 48/* we don't need LOCK_PREFIX since aligned 64-bit writes 49 * are atomic on 586 and newer */ 50 cmpxchg8b (%esi) 51 jne 1b 52 53 ret 54 CFI_ENDPROC 55ENDPROC(atomic64_set_cx8) 56 57ENTRY(atomic64_xchg_cx8) 58 CFI_STARTPROC 59 60 movl %ebx, %eax 61 movl %ecx, %edx 621: 63 LOCK_PREFIX 64 cmpxchg8b (%esi) 65 jne 1b 66 67 ret 68 CFI_ENDPROC 69ENDPROC(atomic64_xchg_cx8) 70 71.macro addsub_return func ins insc 72ENTRY(atomic64_\func\()_return_cx8) 73 CFI_STARTPROC 74 SAVE ebp 75 SAVE ebx 76 SAVE esi 77 SAVE edi 78 79 movl %eax, %esi 80 movl %edx, %edi 81 movl %ecx, %ebp 82 83 read64 %ebp 841: 85 movl %eax, %ebx 86 movl %edx, %ecx 87 \ins\()l %esi, %ebx 88 \insc\()l %edi, %ecx 89 LOCK_PREFIX 90 cmpxchg8b (%ebp) 91 jne 1b 92 9310: 94 movl %ebx, %eax 95 movl %ecx, %edx 96 RESTORE edi 97 RESTORE esi 98 RESTORE ebx 99 RESTORE ebp 100 ret 101 CFI_ENDPROC 102ENDPROC(atomic64_\func\()_return_cx8) 103.endm 104 105addsub_return add add adc 106addsub_return sub sub sbb 107 108.macro incdec_return func ins insc 109ENTRY(atomic64_\func\()_return_cx8) 110 CFI_STARTPROC 111 SAVE ebx 112 113 read64 %esi 1141: 115 movl %eax, %ebx 116 movl %edx, %ecx 117 \ins\()l $1, %ebx 118 \insc\()l $0, %ecx 119 LOCK_PREFIX 120 cmpxchg8b (%esi) 121 jne 1b 122 12310: 124 movl %ebx, %eax 125 movl %ecx, %edx 126 RESTORE ebx 127 ret 128 CFI_ENDPROC 129ENDPROC(atomic64_\func\()_return_cx8) 130.endm 131 132incdec_return inc add adc 133incdec_return dec sub sbb 134 135ENTRY(atomic64_dec_if_positive_cx8) 136 CFI_STARTPROC 137 SAVE ebx 138 139 read64 %esi 1401: 141 movl %eax, %ebx 142 movl %edx, %ecx 143 subl $1, %ebx 144 sbb $0, %ecx 145 js 2f 146 LOCK_PREFIX 147 cmpxchg8b (%esi) 148 jne 1b 149 1502: 151 movl %ebx, %eax 152 movl %ecx, %edx 153 RESTORE ebx 154 ret 155 CFI_ENDPROC 156ENDPROC(atomic64_dec_if_positive_cx8) 157 158ENTRY(atomic64_add_unless_cx8) 159 CFI_STARTPROC 160 SAVE ebp 161 SAVE ebx 162/* these just push these two parameters on the stack */ 163 SAVE edi 164 SAVE esi 165 166 movl %ecx, %ebp 167 movl %eax, %esi 168 movl %edx, %edi 169 170 read64 %ebp 1711: 172 cmpl %eax, 0(%esp) 173 je 4f 1742: 175 movl %eax, %ebx 176 movl %edx, %ecx 177 addl %esi, %ebx 178 adcl %edi, %ecx 179 LOCK_PREFIX 180 cmpxchg8b (%ebp) 181 jne 1b 182 183 movl $1, %eax 1843: 185 addl $8, %esp 186 CFI_ADJUST_CFA_OFFSET -8 187 RESTORE ebx 188 RESTORE ebp 189 ret 1904: 191 cmpl %edx, 4(%esp) 192 jne 2b 193 xorl %eax, %eax 194 jmp 3b 195 CFI_ENDPROC 196ENDPROC(atomic64_add_unless_cx8) 197 198ENTRY(atomic64_inc_not_zero_cx8) 199 CFI_STARTPROC 200 SAVE ebx 201 202 read64 %esi 2031: 204 testl %eax, %eax 205 je 4f 2062: 207 movl %eax, %ebx 208 movl %edx, %ecx 209 addl $1, %ebx 210 adcl $0, %ecx 211 LOCK_PREFIX 212 cmpxchg8b (%esi) 213 jne 1b 214 215 movl $1, %eax 2163: 217 RESTORE ebx 218 ret 2194: 220 testl %edx, %edx 221 jne 2b 222 jmp 3b 223 CFI_ENDPROC 224ENDPROC(atomic64_inc_not_zero_cx8) 225