@@ -82,6 +82,26 @@ glibc {
type: UINT_64
env_alias: LD_HWCAP_MASK
default: HWCAP_IMPORTANT
+ }
+ ifunc {
+ type: STRING
+ env_alias: GLIBC_IFUNC
+ security_level: SXID_IGNORE
+ }
+ non_temporal_threshold {
+ type: SIZE_T
+ env_alias: GLIBC_NON_TEMPORAL_THRESHOLD
+ security_level: SXID_IGNORE
+ }
+ data_cache_size {
+ type: SIZE_T
+ env_alias: GLIBC_DATA_CACHE_SIZE
+ security_level: SXID_IGNORE
+ }
+ shared_cache_size {
+ type: SIZE_T
+ env_alias: GLIBC_SHARED_CACHE_SIZE
+ security_level: SXID_IGNORE
}
}
}
new file mode 100644
@@ -0,0 +1,21 @@
+/* Operating system support for run-time dynamic linker. X86 version.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <config.h>
+#include <sysdeps/x86/cpu-tunables.c>
+#include <sysdeps/unix/sysv/linux/dl-sysdep.c>
@@ -752,6 +752,9 @@ intel_bug_no_cache_info:
#endif
}
+ if (cpu_features->data_cache_size != 0)
+ data = cpu_features->data_cache_size;
+
if (data > 0)
{
__x86_raw_data_cache_size_half = data / 2;
@@ -762,6 +765,9 @@ intel_bug_no_cache_info:
__x86_data_cache_size = data;
}
+ if (cpu_features->shared_cache_size != 0)
+ shared = cpu_features->shared_cache_size;
+
if (shared > 0)
{
__x86_raw_shared_cache_size_half = shared / 2;
@@ -777,7 +783,9 @@ intel_bug_no_cache_info:
store becomes faster on a 8-core processor. This is the 3/4 of the
total shared cache size. */
__x86_shared_non_temporal_threshold
- = __x86_shared_cache_size * threads * 3 / 4;
+ = (cpu_features->non_temporal_threshold != 0
+ ? cpu_features->non_temporal_threshold
+ : __x86_shared_cache_size * threads * 3 / 4);
}
#endif
@@ -20,6 +20,19 @@
#include <cpu-features.h>
#include <dl-hwcap.h>
+#if HAVE_TUNABLES
+# define TUNABLE_NAMESPACE tune
+# include <elf/dl-tunables.h>
+
+extern void TUNABLE_CALLBACK (set_ifunc) (tunable_val_t *)
+ attribute_hidden;
+#else
+# include <string.h>
+# include <unistd.h>
+extern char **__environ attribute_hidden;
+extern void _dl_x86_set_ifunc (const char *) attribute_hidden;
+#endif
+
static void
get_common_indeces (struct cpu_features *cpu_features,
unsigned int *family, unsigned int *model,
@@ -312,6 +325,43 @@ no_cpuid:
cpu_features->model = model;
cpu_features->kind = kind;
+#if HAVE_TUNABLES
+ TUNABLE_GET (ifunc, tunable_val_t *, TUNABLE_CALLBACK (set_ifunc));
+ cpu_features->non_temporal_threshold
+ = TUNABLE_GET (non_temporal_threshold, long int, NULL);
+ cpu_features->data_cache_size
+ = TUNABLE_GET (data_cache_size, long int, NULL);
+ cpu_features->shared_cache_size
+ = TUNABLE_GET (shared_cache_size, long int, NULL);
+#else
+ if (__glibc_likely (__environ != NULL)
+ && !__builtin_expect (__libc_enable_secure, 0))
+ {
+ char **runp = __environ;
+ char *envline;
+
+ while (*runp != NULL)
+ {
+ envline = *runp;
+ if (!DEFAULT_MEMCMP (envline, "GLIBC_IFUNC=",
+ sizeof ("GLIBC_IFUNC=") - 1))
+ _dl_x86_set_ifunc (envline + sizeof ("GLIBC_IFUNC=") - 1);
+ else if (!DEFAULT_MEMCMP (envline,
+ "GLIBC_NON_TEMPORAL_THRESHOLD=", 29))
+ cpu_features->non_temporal_threshold
+ = _dl_strtoul (&envline[29], NULL);
+ else if (!DEFAULT_MEMCMP (envline, "GLIBC_DATA_CACHE_SIZE=", 22))
+ cpu_features->data_cache_size
+ = _dl_strtoul (&envline[22], NULL);
+ else if (!DEFAULT_MEMCMP (envline, "GLIBC_SHARED_CACHE_SIZE=",
+ 24))
+ cpu_features->shared_cache_size
+ = _dl_strtoul (&envline[24], NULL);
+ runp++;
+ }
+ }
+#endif
+
/* Reuse dl_platform, dl_hwcap and dl_hwcap_mask for x86. */
GLRO(dl_platform) = NULL;
GLRO(dl_hwcap) = 0;
@@ -215,6 +215,14 @@ struct cpu_features
unsigned int family;
unsigned int model;
unsigned int feature[FEATURE_INDEX_MAX];
+ /* Data cache size for use in memory and string routines, typically
+ L1 size. */
+ long int data_cache_size;
+ /* Shared cache size for use in memory and string routines, typically
+ L2 or L3 size. */
+ long int shared_cache_size;
+ /* Threshold to use non temporal store. */
+ long int non_temporal_threshold;
};
/* Used from outside of glibc to get access to the CPU features
@@ -228,6 +236,21 @@ extern const struct cpu_features *__get_cpu_features (void)
# define __get_cpu_features() (&GLRO(dl_x86_cpu_features))
# endif
+/* We can't use IFUNC memcmp in init_cpu_features from libc.a since
+ IFUNC must be set up by init_cpu_features. */
+# if defined USE_MULTIARCH && !defined SHARED
+# ifdef __x86_64__
+# define DEFAULT_MEMCMP __memcmp_sse2
+# else
+# define DEFAULT_MEMCMP __memcmp_ia32
+# endif
+extern __typeof (memcmp) DEFAULT_MEMCMP;
+# else
+# define DEFAULT_MEMCMP memcmp
+# endif
+
+/* There is no IFUNC strlen in libc.a. */
+# define DEFAULT_STRLEN strlen
/* Only used directly in cpu-features.c. */
# define CPU_FEATURES_CPU_P(ptr, name) \
new file mode 100644
@@ -0,0 +1,324 @@
+/* CPU feature tuning.
+ This file is part of the GNU C Library.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#if HAVE_TUNABLES
+# define TUNABLE_NAMESPACE tune
+# include <stdbool.h>
+# include <stdint.h>
+# include <elf/dl-tunables.h>
+#endif
+#include <string.h>
+#include <cpu-features.h>
+#include <ldsodefs.h>
+
+#define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
+ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
+ if (!DEFAULT_MEMCMP (f, #name, len)) \
+ { \
+ cpu_features->cpuid[index_cpu_##name].reg_##name \
+ &= ~bit_cpu_##name; \
+ break; \
+ }
+
+/* Disable an ARCH feature NAME. We don't enable an ARCH feature which
+ isn't available. */
+#define CHECK_GLIBC_IFUNC_ARCH_OFF(f, cpu_features, name, len) \
+ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
+ if (!DEFAULT_MEMCMP (f, #name, len)) \
+ { \
+ cpu_features->feature[index_arch_##name] \
+ &= ~bit_arch_##name; \
+ break; \
+ }
+
+/* Enable/disable an ARCH feature NAME. */
+#define CHECK_GLIBC_IFUNC_ARCH_BOTH(f, cpu_features, name, disable, \
+ len) \
+ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
+ if (!DEFAULT_MEMCMP (f, #name, len)) \
+ { \
+ if (disable) \
+ cpu_features->feature[index_arch_##name] \
+ &= ~bit_arch_##name; \
+ else \
+ cpu_features->feature[index_arch_##name] \
+ |= bit_arch_##name; \
+ break; \
+ }
+
+/* Enable/disable an ARCH feature NAME. Enable an ARCH feature only
+ if the ARCH feature NEED is also enabled. */
+#define CHECK_GLIBC_IFUNC_ARCH_NEED_ARCH_BOTH(f, cpu_features, name, \
+ need, disable, len) \
+ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
+ if (!DEFAULT_MEMCMP (f, #name, len)) \
+ { \
+ if (disable) \
+ cpu_features->feature[index_arch_##name] \
+ &= ~bit_arch_##name; \
+ else if (CPU_FEATURES_ARCH_P (cpu_features, need)) \
+ cpu_features->feature[index_arch_##name] \
+ |= bit_arch_##name; \
+ break; \
+ }
+
+/* Enable/disable an ARCH feature NAME. Enable an ARCH feature only
+ if the CPU feature NEED is also enabled. */
+#define CHECK_GLIBC_IFUNC_ARCH_NEED_CPU_BOTH(f, cpu_features, name, \
+ need, disable, len) \
+ _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
+ if (!DEFAULT_MEMCMP (f, #name, len)) \
+ { \
+ if (disable) \
+ cpu_features->feature[index_arch_##name] \
+ &= ~bit_arch_##name; \
+ else if (CPU_FEATURES_CPU_P (cpu_features, need)) \
+ cpu_features->feature[index_arch_##name] \
+ |= bit_arch_##name; \
+ break; \
+ }
+
+#if HAVE_TUNABLES
+static
+#else
+attribute_hidden
+#endif
+void
+_dl_x86_set_ifunc (const char *p)
+{
+ /* The current IFUNC selection is based on microbenchmarks in glibc.
+ It should give the best performance for most workloads. But other
+ choices may have better performance for a particular workload or on
+ the hardware which wasn't available when the selection was made.
+ The environment variable, GLIBC_IFUNC=-xxx,yyy,-zzz...., can be
+ used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature yyy
+ and zzz, where the feature name is case-sensitive and has to match
+ the ones in cpu-features.h. It can be used by glibc developers to
+ tune for a new processor or override the IFUNC selection to improve
+ performance for a particular workload.
+
+ Since all CPU/ARCH features are hardware optimizations without
+ security implication, except for Prefer_MAP_32BIT_EXEC, which can
+ only be disabled, we check GLIBC_IFUNC for programs, including
+ set*id ones.
+
+ NOTE: the IFUNC selection may change over time. Please check all
+ multiarch implementations when experimenting. */
+
+ struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
+ const char *end = p + DEFAULT_STRLEN (p);
+ size_t len;
+
+ do
+ {
+ const char *c, *n;
+ bool disable;
+ size_t nl;
+
+ for (c = p; *c != ','; c++)
+ if (c >= end)
+ break;
+
+ len = c - p;
+ disable = *p == '-';
+ if (disable)
+ {
+ n = p + 1;
+ nl = len - 1;
+ }
+ else
+ {
+ n = p;
+ nl = len;
+ }
+ switch (nl)
+ {
+ default:
+ break;
+ case 3:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
+ }
+ break;
+ case 4:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features, I586, 4);
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features, I686, 4);
+ }
+ break;
+ case 5:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
+ }
+ break;
+ case 6:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
+ }
+ break;
+ case 7:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
+ }
+ break;
+ case 8:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
+ CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
+ }
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features, Slow_BSF,
+ disable, 8);
+ break;
+ case 10:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features, AVX_Usable,
+ 10);
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features, FMA_Usable,
+ 10);
+ }
+ break;
+ case 11:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features, AVX2_Usable,
+ 11);
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features, FMA4_Usable,
+ 11);
+ }
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features, Prefer_ERMS,
+ disable, 11);
+ CHECK_GLIBC_IFUNC_ARCH_NEED_CPU_BOTH (n, cpu_features,
+ Slow_SSE4_2, SSE4_2,
+ disable, 11);
+ break;
+ case 14:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features,
+ AVX512F_Usable, 14);
+ }
+ break;
+ case 15:
+ if (disable)
+ {
+ CHECK_GLIBC_IFUNC_ARCH_OFF (n, cpu_features,
+ AVX512DQ_Usable, 15);
+ }
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features, Fast_Rep_String,
+ disable, 15);
+ break;
+ case 16:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_NEED_ARCH_BOTH
+ (n, cpu_features, Prefer_No_AVX512, AVX512F_Usable,
+ disable, 16);
+ }
+ break;
+ case 18:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features,
+ Fast_Copy_Backward, disable,
+ 18);
+ }
+ break;
+ case 19:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features,
+ Fast_Unaligned_Load, disable,
+ 19);
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features,
+ Fast_Unaligned_Copy, disable,
+ 19);
+ }
+ break;
+ case 20:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_NEED_ARCH_BOTH
+ (n, cpu_features, Prefer_No_VZEROUPPER, AVX_Usable,
+ disable, 20);
+ }
+ break;
+ case 21:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features,
+ Prefer_MAP_32BIT_EXEC, disable,
+ 21);
+ }
+ break;
+ case 23:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_NEED_ARCH_BOTH
+ (n, cpu_features, AVX_Fast_Unaligned_Load, AVX_Usable,
+ disable, 23);
+ }
+ break;
+ case 26:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_NEED_CPU_BOTH
+ (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
+ disable, 26);
+ }
+ break;
+ case 27:
+ {
+ CHECK_GLIBC_IFUNC_ARCH_BOTH (n, cpu_features,
+ Use_dl_runtime_resolve_slow,
+ disable, 27);
+ }
+ break;
+ }
+ p += len + 1;
+ }
+ while (p < end);
+}
+
+#if HAVE_TUNABLES
+attribute_hidden
+void
+TUNABLE_CALLBACK (set_ifunc) (tunable_val_t *valp)
+{
+ _dl_x86_set_ifunc (valp->strval);
+}
+#endif