linux_x86_64.s revision 1472:c18cbe5936b8 
1#
2# Copyright (c) 2004, 2007, Oracle and/or its affiliates. All rights reserved.
3# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4#
5# This code is free software; you can redistribute it and/or modify it
6# under the terms of the GNU General Public License version 2 only, as
7# published by the Free Software Foundation.
8#
9# This code is distributed in the hope that it will be useful, but WITHOUT
10# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12# version 2 for more details (a copy is included in the LICENSE file that
13# accompanied this code).
14#
15# You should have received a copy of the GNU General Public License version
16# 2 along with this work; if not, write to the Free Software Foundation,
17# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18#
19# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20# or visit www.oracle.com if you need additional information or have any
21# questions.
22#
23
24
25        # NOTE WELL!  The _Copy functions are called directly
26	# from server-compiler-generated code via CallLeafNoFP,
27	# which means that they *must* either not use floating
28	# point or use it in the same manner as does the server
29	# compiler.
30
31        .globl _Copy_arrayof_conjoint_bytes
32	.globl _Copy_arrayof_conjoint_jshorts
33        .globl _Copy_conjoint_jshorts_atomic
34        .globl _Copy_arrayof_conjoint_jints
35        .globl _Copy_conjoint_jints_atomic
36        .globl _Copy_arrayof_conjoint_jlongs
37        .globl _Copy_conjoint_jlongs_atomic
38
39	.text
40
41        .globl SafeFetch32, Fetch32PFI, Fetch32Resume
42        .align  16
43        .type   SafeFetch32,@function
44        // Prototype: int SafeFetch32 (int * Adr, int ErrValue)
45SafeFetch32:
46        movl    %esi, %eax
47Fetch32PFI:
48        movl    (%rdi), %eax
49Fetch32Resume:
50        ret
51
52        .globl SafeFetchN, FetchNPFI, FetchNResume
53        .align  16
54        .type   SafeFetchN,@function
55        // Prototype: intptr_t SafeFetchN (intptr_t * Adr, intptr_t ErrValue)
56SafeFetchN:
57        movq    %rsi, %rax
58FetchNPFI:
59        movq    (%rdi), %rax
60FetchNResume:
61        ret
62
63        .globl SpinPause
64        .align 16
65        .type  SpinPause,@function
66SpinPause:
67        rep
68        nop
69        movq   $1, %rax
70        ret
71
72        # Support for void Copy::arrayof_conjoint_bytes(void* from,
73        #                                               void* to,
74        #                                               size_t count)
75        # rdi - from
76        # rsi - to
77        # rdx - count, treated as ssize_t
78        #
79        .p2align 4,,15
80	.type    _Copy_arrayof_conjoint_bytes,@function
81_Copy_arrayof_conjoint_bytes:
82        movq     %rdx,%r8             # byte count
83        shrq     $3,%rdx              # qword count
84        cmpq     %rdi,%rsi
85        leaq     -1(%rdi,%r8,1),%rax  # from + bcount*1 - 1
86        jbe      acb_CopyRight
87        cmpq     %rax,%rsi
88        jbe      acb_CopyLeft
89acb_CopyRight:
90        leaq     -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
91        leaq     -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
92        negq     %rdx
93        jmp      7f
94        .p2align 4,,15
951:      movq     8(%rax,%rdx,8),%rsi
96        movq     %rsi,8(%rcx,%rdx,8)
97        addq     $1,%rdx
98        jnz      1b
992:      testq    $4,%r8               # check for trailing dword
100        jz       3f
101        movl     8(%rax),%esi         # copy trailing dword
102        movl     %esi,8(%rcx)
103        addq     $4,%rax
104        addq     $4,%rcx              # original %rsi is trashed, so we
105                                      #  can't use it as a base register
1063:      testq    $2,%r8               # check for trailing word
107        jz       4f
108        movw     8(%rax),%si          # copy trailing word
109        movw     %si,8(%rcx)
110        addq     $2,%rcx
1114:      testq    $1,%r8               # check for trailing byte
112        jz       5f
113        movb     -1(%rdi,%r8,1),%al   # copy trailing byte
114        movb     %al,8(%rcx)
1155:      ret
116        .p2align 4,,15
1176:      movq     -24(%rax,%rdx,8),%rsi
118        movq     %rsi,-24(%rcx,%rdx,8)
119        movq     -16(%rax,%rdx,8),%rsi
120        movq     %rsi,-16(%rcx,%rdx,8)
121        movq     -8(%rax,%rdx,8),%rsi
122        movq     %rsi,-8(%rcx,%rdx,8)
123        movq     (%rax,%rdx,8),%rsi
124        movq     %rsi,(%rcx,%rdx,8)
1257:      addq     $4,%rdx
126        jle      6b
127        subq     $4,%rdx
128        jl       1b
129        jmp      2b
130acb_CopyLeft:
131        testq    $1,%r8               # check for trailing byte
132        jz       1f
133        movb     -1(%rdi,%r8,1),%cl   # copy trailing byte
134        movb     %cl,-1(%rsi,%r8,1)
135        subq     $1,%r8               # adjust for possible trailing word
1361:      testq    $2,%r8               # check for trailing word
137        jz       2f
138        movw     -2(%rdi,%r8,1),%cx   # copy trailing word
139        movw     %cx,-2(%rsi,%r8,1)
1402:      testq    $4,%r8               # check for trailing dword
141        jz       5f
142        movl     (%rdi,%rdx,8),%ecx   # copy trailing dword
143        movl     %ecx,(%rsi,%rdx,8)
144        jmp      5f
145        .p2align 4,,15
1463:      movq     -8(%rdi,%rdx,8),%rcx
147        movq     %rcx,-8(%rsi,%rdx,8)
148        subq     $1,%rdx
149        jnz      3b
150        ret
151        .p2align 4,,15
1524:      movq     24(%rdi,%rdx,8),%rcx
153        movq     %rcx,24(%rsi,%rdx,8)
154        movq     16(%rdi,%rdx,8),%rcx
155        movq     %rcx,16(%rsi,%rdx,8)
156        movq     8(%rdi,%rdx,8),%rcx
157        movq     %rcx,8(%rsi,%rdx,8)
158        movq     (%rdi,%rdx,8),%rcx
159        movq     %rcx,(%rsi,%rdx,8)
1605:      subq     $4,%rdx
161        jge      4b
162        addq     $4,%rdx
163        jg       3b
164        ret
165
166        # Support for void Copy::arrayof_conjoint_jshorts(void* from,
167        #                                                 void* to,
168        #                                                 size_t count)
169        # Equivalent to
170        #   conjoint_jshorts_atomic
171        #
172        # If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
173        # let the hardware handle it.  The tow or four words within dwords
174        # or qwords that span cache line boundaries will still be loaded
175        # and stored atomically.
176        #
177        # rdi - from
178        # rsi - to
179        # rdx - count, treated as ssize_t
180        #
181        .p2align 4,,15
182	.type    _Copy_arrayof_conjoint_jshorts,@function
183	.type    _Copy_conjoint_jshorts_atomic,@function
184_Copy_arrayof_conjoint_jshorts:
185_Copy_conjoint_jshorts_atomic:
186        movq     %rdx,%r8             # word count
187        shrq     $2,%rdx              # qword count
188        cmpq     %rdi,%rsi
189        leaq     -2(%rdi,%r8,2),%rax  # from + wcount*2 - 2
190        jbe      acs_CopyRight
191        cmpq     %rax,%rsi
192        jbe      acs_CopyLeft
193acs_CopyRight:
194        leaq     -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
195        leaq     -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
196        negq     %rdx
197        jmp      6f
1981:      movq     8(%rax,%rdx,8),%rsi
199        movq     %rsi,8(%rcx,%rdx,8)
200        addq     $1,%rdx
201        jnz      1b
2022:      testq    $2,%r8               # check for trailing dword
203        jz       3f
204        movl     8(%rax),%esi         # copy trailing dword
205        movl     %esi,8(%rcx)
206        addq     $4,%rcx              # original %rsi is trashed, so we
207                                      #  can't use it as a base register
2083:      testq    $1,%r8               # check for trailing word
209        jz       4f
210        movw     -2(%rdi,%r8,2),%si   # copy trailing word
211        movw     %si,8(%rcx)
2124:      ret
213        .p2align 4,,15
2145:      movq     -24(%rax,%rdx,8),%rsi
215        movq     %rsi,-24(%rcx,%rdx,8)
216        movq     -16(%rax,%rdx,8),%rsi
217        movq     %rsi,-16(%rcx,%rdx,8)
218        movq     -8(%rax,%rdx,8),%rsi
219        movq     %rsi,-8(%rcx,%rdx,8)
220        movq     (%rax,%rdx,8),%rsi
221        movq     %rsi,(%rcx,%rdx,8)
2226:      addq     $4,%rdx
223        jle      5b
224        subq     $4,%rdx
225        jl       1b
226        jmp      2b
227acs_CopyLeft:
228        testq    $1,%r8               # check for trailing word
229        jz       1f
230        movw     -2(%rdi,%r8,2),%cx   # copy trailing word
231        movw     %cx,-2(%rsi,%r8,2)
2321:      testq    $2,%r8               # check for trailing dword
233        jz       4f
234        movl     (%rdi,%rdx,8),%ecx   # copy trailing dword
235        movl     %ecx,(%rsi,%rdx,8)
236        jmp      4f
2372:      movq     -8(%rdi,%rdx,8),%rcx
238        movq     %rcx,-8(%rsi,%rdx,8)
239        subq     $1,%rdx
240        jnz      2b
241        ret
242        .p2align 4,,15
2433:      movq     24(%rdi,%rdx,8),%rcx
244        movq     %rcx,24(%rsi,%rdx,8)
245        movq     16(%rdi,%rdx,8),%rcx
246        movq     %rcx,16(%rsi,%rdx,8)
247        movq     8(%rdi,%rdx,8),%rcx
248        movq     %rcx,8(%rsi,%rdx,8)
249        movq     (%rdi,%rdx,8),%rcx
250        movq     %rcx,(%rsi,%rdx,8)
2514:      subq     $4,%rdx
252        jge      3b
253        addq     $4,%rdx
254        jg       2b
255        ret
256
257        # Support for void Copy::arrayof_conjoint_jints(jint* from,
258        #                                               jint* to,
259        #                                               size_t count)
260        # Equivalent to
261        #   conjoint_jints_atomic
262        #
263        # If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
264        # the hardware handle it.  The two dwords within qwords that span
265        # cache line boundaries will still be loaded and stored atomically.
266        #
267        # rdi - from
268        # rsi - to
269        # rdx - count, treated as ssize_t
270        #
271        .p2align 4,,15
272	.type    _Copy_arrayof_conjoint_jints,@function
273	.type    _Copy_conjoint_jints_atomic,@function
274_Copy_arrayof_conjoint_jints:
275_Copy_conjoint_jints_atomic:
276        movq     %rdx,%r8             # dword count
277        shrq     %rdx                 # qword count
278        cmpq     %rdi,%rsi
279        leaq     -4(%rdi,%r8,4),%rax  # from + dcount*4 - 4
280        jbe      aci_CopyRight
281        cmpq     %rax,%rsi
282        jbe      aci_CopyLeft
283aci_CopyRight:
284        leaq     -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
285        leaq     -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
286        negq     %rdx
287        jmp      5f
288        .p2align 4,,15
2891:      movq     8(%rax,%rdx,8),%rsi
290        movq     %rsi,8(%rcx,%rdx,8)
291        addq     $1,%rdx
292        jnz       1b
2932:      testq    $1,%r8               # check for trailing dword
294        jz       3f
295        movl     8(%rax),%esi         # copy trailing dword
296        movl     %esi,8(%rcx)
2973:      ret
298        .p2align 4,,15
2994:      movq     -24(%rax,%rdx,8),%rsi
300        movq     %rsi,-24(%rcx,%rdx,8)
301        movq     -16(%rax,%rdx,8),%rsi
302        movq     %rsi,-16(%rcx,%rdx,8)
303        movq     -8(%rax,%rdx,8),%rsi
304        movq     %rsi,-8(%rcx,%rdx,8)
305        movq     (%rax,%rdx,8),%rsi
306        movq     %rsi,(%rcx,%rdx,8)
3075:      addq     $4,%rdx
308        jle      4b
309        subq     $4,%rdx
310        jl       1b
311        jmp      2b
312aci_CopyLeft:
313        testq    $1,%r8               # check for trailing dword
314        jz       3f
315        movl     -4(%rdi,%r8,4),%ecx  # copy trailing dword
316        movl     %ecx,-4(%rsi,%r8,4)
317        jmp      3f
3181:      movq     -8(%rdi,%rdx,8),%rcx
319        movq     %rcx,-8(%rsi,%rdx,8)
320        subq     $1,%rdx
321        jnz      1b
322        ret
323        .p2align 4,,15
3242:      movq     24(%rdi,%rdx,8),%rcx
325        movq     %rcx,24(%rsi,%rdx,8)
326        movq     16(%rdi,%rdx,8),%rcx
327        movq     %rcx,16(%rsi,%rdx,8)
328        movq     8(%rdi,%rdx,8),%rcx
329        movq     %rcx,8(%rsi,%rdx,8)
330        movq     (%rdi,%rdx,8),%rcx
331        movq     %rcx,(%rsi,%rdx,8)
3323:      subq     $4,%rdx
333        jge      2b
334        addq     $4,%rdx
335        jg       1b
336        ret
337
338        # Support for void Copy::arrayof_conjoint_jlongs(jlong* from,
339        #                                                jlong* to,
340        #                                                size_t count)
341        # Equivalent to
342        #   conjoint_jlongs_atomic
343        #   arrayof_conjoint_oops
344        #   conjoint_oops_atomic
345        #
346        # rdi - from
347        # rsi - to
348        # rdx - count, treated as ssize_t
349        #
350        .p2align 4,,15
351	.type    _Copy_arrayof_conjoint_jlongs,@function
352	.type    _Copy_conjoint_jlongs_atomic,@function
353_Copy_arrayof_conjoint_jlongs:
354_Copy_conjoint_jlongs_atomic:
355        cmpq     %rdi,%rsi
356        leaq     -8(%rdi,%rdx,8),%rax # from + count*8 - 8
357        jbe      acl_CopyRight
358        cmpq     %rax,%rsi
359        jbe      acl_CopyLeft
360acl_CopyRight:
361        leaq     -8(%rsi,%rdx,8),%rcx # to + count*8 - 8
362        negq     %rdx
363        jmp      3f
3641:      movq     8(%rax,%rdx,8),%rsi
365        movq     %rsi,8(%rcx,%rdx,8)
366        addq     $1,%rdx
367        jnz      1b
368        ret
369        .p2align 4,,15
3702:      movq     -24(%rax,%rdx,8),%rsi
371        movq     %rsi,-24(%rcx,%rdx,8)
372        movq     -16(%rax,%rdx,8),%rsi
373        movq     %rsi,-16(%rcx,%rdx,8)
374        movq     -8(%rax,%rdx,8),%rsi
375        movq     %rsi,-8(%rcx,%rdx,8)
376        movq     (%rax,%rdx,8),%rsi
377        movq     %rsi,(%rcx,%rdx,8)
3783:      addq     $4,%rdx
379        jle      2b
380        subq     $4,%rdx
381        jl       1b
382        ret
3834:      movq     -8(%rdi,%rdx,8),%rcx
384        movq     %rcx,-8(%rsi,%rdx,8)
385        subq     $1,%rdx
386        jnz      4b
387        ret
388        .p2align 4,,15
3895:      movq     24(%rdi,%rdx,8),%rcx
390        movq     %rcx,24(%rsi,%rdx,8)
391        movq     16(%rdi,%rdx,8),%rcx
392        movq     %rcx,16(%rsi,%rdx,8)
393        movq     8(%rdi,%rdx,8),%rcx
394        movq     %rcx,8(%rsi,%rdx,8)
395        movq     (%rdi,%rdx,8),%rcx
396        movq     %rcx,(%rsi,%rdx,8)
397acl_CopyLeft:
398        subq     $4,%rdx
399        jge      5b
400        addq     $4,%rdx
401        jg       4b
402        ret
403