| support.S (338427) | support.S (347568) |
|---|---|
| 1/*- | 1/*- |
| 2 * Copyright (c) 2018-2019 The FreeBSD Foundation |
|
| 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1993 The Regents of the University of California. 4 * All rights reserved. 5 * | 3 * Copyright (c) 2003 Peter Wemm. 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * |
| 7 * Portions of this software were developed by 8 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 9 * the FreeBSD Foundation. 10 * |
|
| 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. --- 8 unchanged lines hidden (view full) --- 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * | 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. --- 8 unchanged lines hidden (view full) --- 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * |
| 30 * $FreeBSD: stable/11/sys/amd64/amd64/support.S 338427 2018-09-02 10:51:31Z kib $ | 35 * $FreeBSD: stable/11/sys/amd64/amd64/support.S 347568 2019-05-14 17:05:02Z kib $ |
| 31 */ 32 33#include "opt_ddb.h" 34 35#include <machine/asmacros.h> 36#include <machine/specialreg.h> 37#include <machine/pmap.h> 38 --- 881 unchanged lines hidden (view full) --- 920 /* pass 2: Read each cache line. */ 9212: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 922 addq $64, %rcx 923 jne 2b 924 lfence 925 ret 926#undef L1D_FLUSH_SIZE 927END(flush_l1d_sw) | 36 */ 37 38#include "opt_ddb.h" 39 40#include <machine/asmacros.h> 41#include <machine/specialreg.h> 42#include <machine/pmap.h> 43 --- 881 unchanged lines hidden (view full) --- 925 /* pass 2: Read each cache line. */ 9262: movb L1D_FLUSH_SIZE(%r9, %rcx), %al 927 addq $64, %rcx 928 jne 2b 929 lfence 930 ret 931#undef L1D_FLUSH_SIZE 932END(flush_l1d_sw) |
| 933 934ENTRY(mds_handler_void) 935 retq 936END(mds_handler_void) 937 938ENTRY(mds_handler_verw) 939 subq $8, %rsp 940 movw %ds, (%rsp) 941 verw (%rsp) 942 addq $8, %rsp 943 retq 944END(mds_handler_verw) 945 946ENTRY(mds_handler_ivb) 947 pushq %rax 948 pushq %rdx 949 pushq %rcx 950 951 movq %cr0, %rax 952 testb $CR0_TS, %al 953 je 1f 954 clts 9551: movq PCPU(MDS_BUF), %rdx 956 movdqa %xmm0, PCPU(MDS_TMP) 957 pxor %xmm0, %xmm0 958 959 lfence 960 orpd (%rdx), %xmm0 961 orpd (%rdx), %xmm0 962 mfence 963 movl $40, %ecx 964 addq $16, %rdx 9652: movntdq %xmm0, (%rdx) 966 addq $16, %rdx 967 decl %ecx 968 jnz 2b 969 mfence 970 971 movdqa PCPU(MDS_TMP),%xmm0 972 testb $CR0_TS, %al 973 je 3f 974 movq %rax, %cr0 9753: popq %rcx 976 popq %rdx 977 popq %rax 978 retq 979END(mds_handler_ivb) 980 981ENTRY(mds_handler_bdw) 982 pushq %rax 983 pushq %rbx 984 pushq %rcx 985 pushq %rdi 986 pushq %rsi 987 988 movq %cr0, %rax 989 testb $CR0_TS, %al 990 je 1f 991 clts 9921: movq PCPU(MDS_BUF), %rbx 993 movdqa %xmm0, PCPU(MDS_TMP) 994 pxor %xmm0, %xmm0 995 996 movq %rbx, %rdi 997 movq %rbx, %rsi 998 movl $40, %ecx 9992: movntdq %xmm0, (%rbx) 1000 addq $16, %rbx 1001 decl %ecx 1002 jnz 2b 1003 mfence 1004 movl $1536, %ecx 1005 rep; movsb 1006 lfence 1007 1008 movdqa PCPU(MDS_TMP),%xmm0 1009 testb $CR0_TS, %al 1010 je 3f 1011 movq %rax, %cr0 10123: popq %rsi 1013 popq %rdi 1014 popq %rcx 1015 popq %rbx 1016 popq %rax 1017 retq 1018END(mds_handler_bdw) 1019 1020ENTRY(mds_handler_skl_sse) 1021 pushq %rax 1022 pushq %rdx 1023 pushq %rcx 1024 pushq %rdi 1025 1026 movq %cr0, %rax 1027 testb $CR0_TS, %al 1028 je 1f 1029 clts 10301: movq PCPU(MDS_BUF), %rdi 1031 movq PCPU(MDS_BUF64), %rdx 1032 movdqa %xmm0, PCPU(MDS_TMP) 1033 pxor %xmm0, %xmm0 1034 1035 lfence 1036 orpd (%rdx), %xmm0 1037 orpd (%rdx), %xmm0 1038 xorl %eax, %eax 10392: clflushopt 5376(%rdi, %rax, 8) 1040 addl $8, %eax 1041 cmpl $8 * 12, %eax 1042 jb 2b 1043 sfence 1044 movl $6144, %ecx 1045 xorl %eax, %eax 1046 rep; stosb 1047 mfence 1048 1049 movdqa PCPU(MDS_TMP), %xmm0 1050 testb $CR0_TS, %al 1051 je 3f 1052 movq %rax, %cr0 10533: popq %rdi 1054 popq %rcx 1055 popq %rdx 1056 popq %rax 1057 retq 1058END(mds_handler_skl_sse) 1059 1060ENTRY(mds_handler_skl_avx) 1061 pushq %rax 1062 pushq %rdx 1063 pushq %rcx 1064 pushq %rdi 1065 1066 movq %cr0, %rax 1067 testb $CR0_TS, %al 1068 je 1f 1069 clts 10701: movq PCPU(MDS_BUF), %rdi 1071 movq PCPU(MDS_BUF64), %rdx 1072 vmovdqa %ymm0, PCPU(MDS_TMP) 1073 vpxor %ymm0, %ymm0, %ymm0 1074 1075 lfence 1076 vorpd (%rdx), %ymm0, %ymm0 1077 vorpd (%rdx), %ymm0, %ymm0 1078 xorl %eax, %eax 10792: clflushopt 5376(%rdi, %rax, 8) 1080 addl $8, %eax 1081 cmpl $8 * 12, %eax 1082 jb 2b 1083 sfence 1084 movl $6144, %ecx 1085 xorl %eax, %eax 1086 rep; stosb 1087 mfence 1088 1089 vmovdqa PCPU(MDS_TMP), %ymm0 1090 testb $CR0_TS, %al 1091 je 3f 1092 movq %rax, %cr0 10933: popq %rdi 1094 popq %rcx 1095 popq %rdx 1096 popq %rax 1097 retq 1098END(mds_handler_skl_avx) 1099 1100ENTRY(mds_handler_skl_avx512) 1101 pushq %rax 1102 pushq %rdx 1103 pushq %rcx 1104 pushq %rdi 1105 1106 movq %cr0, %rax 1107 testb $CR0_TS, %al 1108 je 1f 1109 clts 11101: movq PCPU(MDS_BUF), %rdi 1111 movq PCPU(MDS_BUF64), %rdx 1112 vmovdqa64 %zmm0, PCPU(MDS_TMP) 1113 vpxor %zmm0, %zmm0, %zmm0 1114 1115 lfence 1116 vorpd (%rdx), %zmm0, %zmm0 1117 vorpd (%rdx), %zmm0, %zmm0 1118 xorl %eax, %eax 11192: clflushopt 5376(%rdi, %rax, 8) 1120 addl $8, %eax 1121 cmpl $8 * 12, %eax 1122 jb 2b 1123 sfence 1124 movl $6144, %ecx 1125 xorl %eax, %eax 1126 rep; stosb 1127 mfence 1128 1129 vmovdqa64 PCPU(MDS_TMP), %zmm0 1130 testb $CR0_TS, %al 1131 je 3f 1132 movq %rax, %cr0 11333: popq %rdi 1134 popq %rcx 1135 popq %rdx 1136 popq %rax 1137 retq 1138END(mds_handler_skl_avx512) 1139 1140ENTRY(mds_handler_silvermont) 1141 pushq %rax 1142 pushq %rdx 1143 pushq %rcx 1144 1145 movq %cr0, %rax 1146 testb $CR0_TS, %al 1147 je 1f 1148 clts 11491: movq PCPU(MDS_BUF), %rdx 1150 movdqa %xmm0, PCPU(MDS_TMP) 1151 pxor %xmm0, %xmm0 1152 1153 movl $16, %ecx 11542: movntdq %xmm0, (%rdx) 1155 addq $16, %rdx 1156 decl %ecx 1157 jnz 2b 1158 mfence 1159 1160 movdqa PCPU(MDS_TMP),%xmm0 1161 testb $CR0_TS, %al 1162 je 3f 1163 movq %rax, %cr0 11643: popq %rcx 1165 popq %rdx 1166 popq %rax 1167 retq 1168END(mds_handler_silvermont) |
|