1/* x86 CPU feature tuning.
2 This file is part of the GNU C Library.
3 Copyright (C) 2017-2020 Free Software Foundation, Inc.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#if HAVE_TUNABLES
20# define TUNABLE_NAMESPACE cpu
21# include <stdbool.h>
22# include <stdint.h>
23# include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */
24# include <elf/dl-tunables.h>
25# include <string.h>
26# include <cpu-features.h>
27# include <ldsodefs.h>
28
29/* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
30 since IFUNC must be set up by init_cpu_features. */
31# if defined USE_MULTIARCH && !defined SHARED
32# ifdef __x86_64__
33# define DEFAULT_MEMCMP __memcmp_sse2
34# else
35# define DEFAULT_MEMCMP __memcmp_ia32
36# endif
37extern __typeof (memcmp) DEFAULT_MEMCMP;
38# else
39# define DEFAULT_MEMCMP memcmp
40# endif
41
42# define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
43 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
44 if (!DEFAULT_MEMCMP (f, #name, len)) \
45 { \
46 CPU_FEATURE_UNSET (cpu_features, name) \
47 break; \
48 }
49
50/* Disable a preferred feature NAME. We don't enable a preferred feature
51 which isn't available. */
52# define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \
53 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
54 if (!DEFAULT_MEMCMP (f, #name, len)) \
55 { \
56 cpu_features->preferred[index_arch_##name] \
57 &= ~bit_arch_##name; \
58 break; \
59 }
60
61/* Enable/disable a preferred feature NAME. */
62# define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \
63 disable, len) \
64 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
65 if (!DEFAULT_MEMCMP (f, #name, len)) \
66 { \
67 if (disable) \
68 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
69 else \
70 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
71 break; \
72 }
73
74/* Enable/disable a preferred feature NAME. Enable a preferred feature
75 only if the feature NEED is usable. */
76# define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \
77 need, disable, len) \
78 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
79 if (!DEFAULT_MEMCMP (f, #name, len)) \
80 { \
81 if (disable) \
82 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
83 else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \
84 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
85 break; \
86 }
87
88attribute_hidden
89void
90TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
91{
92 /* The current IFUNC selection is based on microbenchmarks in glibc.
93 It should give the best performance for most workloads. But other
94 choices may have better performance for a particular workload or on
95 the hardware which wasn't available when the selection was made.
96 The environment variable:
97
98 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
99
100 can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
101 yyy and zzz, where the feature name is case-sensitive and has to
102 match the ones in cpu-features.h. It can be used by glibc developers
103 to tune for a new processor or override the IFUNC selection to
104 improve performance for a particular workload.
105
106 NOTE: the IFUNC selection may change over time. Please check all
107 multiarch implementations when experimenting. */
108
109 const char *p = valp->strval;
110 struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
111 size_t len;
112
113 do
114 {
115 const char *c, *n;
116 bool disable;
117 size_t nl;
118
119 for (c = p; *c != ','; c++)
120 if (*c == '\0')
121 break;
122
123 len = c - p;
124 disable = *p == '-';
125 if (disable)
126 {
127 n = p + 1;
128 nl = len - 1;
129 }
130 else
131 {
132 n = p;
133 nl = len;
134 }
135 switch (nl)
136 {
137 default:
138 break;
139 case 3:
140 if (disable)
141 {
142 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
143 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
144 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
145 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
146 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
147 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
148 }
149 break;
150 case 4:
151 if (disable)
152 {
153 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
154 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
155 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
156 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
157 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
158 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
159 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
160 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
161 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
162 }
163 break;
164 case 5:
165 if (disable)
166 {
167 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
168 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
169 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
170 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
171 }
172 break;
173 case 6:
174 if (disable)
175 {
176 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
177 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
178 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
179 if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
180 {
181 /* Update xsave_state_size to XSAVE state size. */
182 cpu_features->xsave_state_size
183 = cpu_features->xsave_state_full_size;
184 CPU_FEATURE_UNSET (cpu_features, XSAVEC);
185 }
186 }
187 break;
188 case 7:
189 if (disable)
190 {
191 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
192 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
193 }
194 break;
195 case 8:
196 if (disable)
197 {
198 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
199 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
200 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
201 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
202 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
203 CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
204 }
205 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
206 disable, 8);
207 break;
208 case 11:
209 {
210 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
211 Prefer_ERMS,
212 disable, 11);
213 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
214 Prefer_FSRM,
215 disable, 11);
216 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
217 Slow_SSE4_2,
218 SSE4_2,
219 disable, 11);
220 }
221 break;
222 case 15:
223 {
224 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
225 Fast_Rep_String,
226 disable, 15);
227 }
228 break;
229 case 16:
230 {
231 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
232 (n, cpu_features, Prefer_No_AVX512, AVX512F,
233 disable, 16);
234 }
235 break;
236 case 18:
237 {
238 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
239 Fast_Copy_Backward,
240 disable, 18);
241 }
242 break;
243 case 19:
244 {
245 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
246 Fast_Unaligned_Load,
247 disable, 19);
248 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
249 Fast_Unaligned_Copy,
250 disable, 19);
251 }
252 break;
253 case 20:
254 {
255 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
256 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
257 20);
258 }
259 break;
260 case 21:
261 {
262 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
263 Prefer_MAP_32BIT_EXEC,
264 disable, 21);
265 }
266 break;
267 case 23:
268 {
269 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
270 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
271 disable, 23);
272 }
273 break;
274 case 24:
275 {
276 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
277 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
278 disable, 24);
279 }
280 break;
281 case 26:
282 {
283 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
284 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
285 disable, 26);
286 }
287 break;
288 }
289 p += len + 1;
290 }
291 while (*p != '\0');
292}
293
294# if CET_ENABLED
295
296attribute_hidden
297void
298TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
299{
300 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
301 GL(dl_x86_feature_control).ibt = cet_always_on;
302 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
303 GL(dl_x86_feature_control).ibt = cet_always_off;
304 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
305 sizeof ("permissive")) == 0)
306 GL(dl_x86_feature_control).ibt = cet_permissive;
307}
308
309attribute_hidden
310void
311TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
312{
313 if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
314 GL(dl_x86_feature_control).shstk = cet_always_on;
315 else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
316 GL(dl_x86_feature_control).shstk = cet_always_off;
317 else if (DEFAULT_MEMCMP (valp->strval, "permissive",
318 sizeof ("permissive")) == 0)
319 GL(dl_x86_feature_control).shstk = cet_permissive;
320}
321# endif
322#endif
323