elf, nptl: Initialize static TLS directly in ld.so
Checks
Context |
Check |
Description |
dj/dj-TryBot-apply_patch |
success
|
Patch apply succeeded
|
Commit Message
The stack list is available in ld.so since commit
1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list
variables into _rtld_global"), so it's possible to walk the stack
list directly in ld.so and perform the initialization there.
This eliminates an unprotected function pointer from _rtld_global
and reduces the libpthread initialization code.
---
elf/dl-open.c | 2 +-
elf/dl-reloc.c | 5 +++--
elf/dl-support.c | 3 +--
elf/dl-tls.c | 39 +++++++++++++++++++++++++++++++++++++++
elf/rtld.c | 2 ++
nptl/allocatestack.c | 35 -----------------------------------
nptl/nptl-init.c | 2 --
nptl/pthreadP.h | 2 --
sysdeps/generic/ldsodefs.h | 19 +++++++++++++++++++
9 files changed, 65 insertions(+), 44 deletions(-)
Comments
On Tue, May 4, 2021 at 12:01 PM Florian Weimer via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>
> The stack list is available in ld.so since commit
> 1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list
> variables into _rtld_global"), so it's possible to walk the stack
> list directly in ld.so and perform the initialization there.
>
> This eliminates an unprotected function pointer from _rtld_global
> and reduces the libpthread initialization code.
>
> ---
> elf/dl-open.c | 2 +-
> elf/dl-reloc.c | 5 +++--
> elf/dl-support.c | 3 +--
> elf/dl-tls.c | 39 +++++++++++++++++++++++++++++++++++++++
> elf/rtld.c | 2 ++
> nptl/allocatestack.c | 35 -----------------------------------
> nptl/nptl-init.c | 2 --
> nptl/pthreadP.h | 2 --
> sysdeps/generic/ldsodefs.h | 19 +++++++++++++++++++
> 9 files changed, 65 insertions(+), 44 deletions(-)
>
> diff --git a/elf/dl-open.c b/elf/dl-open.c
> index ab7aaa345e..09f0df7d38 100644
> --- a/elf/dl-open.c
> +++ b/elf/dl-open.c
> @@ -426,7 +426,7 @@ TLS generation counter wrapped! Please report this."));
> _dl_update_slotinfo (imap->l_tls_modid);
> #endif
>
> - GL(dl_init_static_tls) (imap);
> + dl_init_static_tls (imap);
> assert (imap->l_need_tls_init == 0);
> }
> }
> diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c
> index c2df26deea..bb9ca1a101 100644
> --- a/elf/dl-reloc.c
> +++ b/elf/dl-reloc.c
> @@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional)
> (void) _dl_update_slotinfo (map->l_tls_modid);
> #endif
>
> - GL(dl_init_static_tls) (map);
> + dl_init_static_tls (map);
> }
> else
> map->l_need_tls_init = 1;
> @@ -141,6 +141,7 @@ cannot allocate memory in static TLS block"));
> }
> }
>
> +#if !THREAD_GSCOPE_IN_TCB
> /* Initialize static TLS area and DTV for current (only) thread.
> libpthread implementations should provide their own hook
> to handle all threads. */
> @@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map)
> memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
> '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
> }
> -
> +#endif /* !THREAD_GSCOPE_IN_TCB */
>
> void
> _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],
> diff --git a/elf/dl-support.c b/elf/dl-support.c
> index 7fc2ee78e2..f966a2e7cd 100644
> --- a/elf/dl-support.c
> +++ b/elf/dl-support.c
> @@ -138,8 +138,6 @@ void *_dl_random;
> #include <dl-procruntime.c>
> #include <dl-procinfo.c>
>
> -void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
> -
> size_t _dl_pagesize = EXEC_PAGESIZE;
>
> size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ;
> @@ -197,6 +195,7 @@ list_t _dl_stack_user;
> int _dl_stack_cache_lock;
> #else
> int _dl_thread_gscope_count;
> +void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
> #endif
> struct dl_scope_free_list *_dl_scope_free_list;
>
> diff --git a/elf/dl-tls.c b/elf/dl-tls.c
> index f8b32b3ecb..6baff0c1ea 100644
> --- a/elf/dl-tls.c
> +++ b/elf/dl-tls.c
> @@ -29,6 +29,10 @@
> #include <dl-tls.h>
> #include <ldsodefs.h>
>
> +#if THREAD_GSCOPE_IN_TCB
> +# include <list.h>
> +#endif
> +
> #define TUNABLE_NAMESPACE rtld
> #include <dl-tunables.h>
>
> @@ -1005,3 +1009,38 @@ cannot create TLS data structures"));
> listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
> }
> }
> +
> +#if THREAD_GSCOPE_IN_TCB
> +static inline void __attribute__((always_inline))
> +init_one_static_tls (struct pthread *curp, struct link_map *map)
> +{
> +# if TLS_TCB_AT_TP
> + void *dest = (char *) curp - map->l_tls_offset;
> +# elif TLS_DTV_AT_TP
> + void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
> +# else
> +# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
> +# endif
> +
> + /* Initialize the memory. */
> + memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
> + '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
> +}
> +
> +void
> +_dl_init_static_tls (struct link_map *map)
> +{
> + lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
> +
> + /* Iterate over the list with system-allocated threads first. */
> + list_t *runp;
> + list_for_each (runp, &GL (dl_stack_used))
> + init_one_static_tls (list_entry (runp, struct pthread, list), map);
> +
> + /* Now the list with threads using user-allocated stacks. */
> + list_for_each (runp, &GL (dl_stack_user))
> + init_one_static_tls (list_entry (runp, struct pthread, list), map);
> +
> + lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
> +}
> +#endif /* THREAD_GSCOPE_IN_TCB */
> diff --git a/elf/rtld.c b/elf/rtld.c
> index 34879016ad..ad325d4c10 100644
> --- a/elf/rtld.c
> +++ b/elf/rtld.c
> @@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr,
> struct dl_main_state state;
> dl_main_state_init (&state);
>
> +#if !THREAD_GSCOPE_IN_TCB
> GL(dl_init_static_tls) = &_dl_nothread_init_static_tls;
> +#endif
>
> #if defined SHARED && defined _LIBC_REENTRANT \
> && defined __rtld_lock_default_lock_recursive
> diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
> index f1270c3109..8aaba088b1 100644
> --- a/nptl/allocatestack.c
> +++ b/nptl/allocatestack.c
> @@ -951,38 +951,3 @@ __reclaim_stacks (void)
> GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
> __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
> }
> -
> -
> -static inline void __attribute__((always_inline))
> -init_one_static_tls (struct pthread *curp, struct link_map *map)
> -{
> -# if TLS_TCB_AT_TP
> - void *dest = (char *) curp - map->l_tls_offset;
> -# elif TLS_DTV_AT_TP
> - void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
> -# else
> -# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
> -# endif
> -
> - /* Initialize the memory. */
> - memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
> - '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
> -}
> -
> -void
> -attribute_hidden
> -__pthread_init_static_tls (struct link_map *map)
> -{
> - lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
> -
> - /* Iterate over the list with system-allocated threads first. */
> - list_t *runp;
> - list_for_each (runp, &GL (dl_stack_used))
> - init_one_static_tls (list_entry (runp, struct pthread, list), map);
> -
> - /* Now the list with threads using user-allocated stacks. */
> - list_for_each (runp, &GL (dl_stack_user))
> - init_one_static_tls (list_entry (runp, struct pthread, list), map);
> -
> - lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
> -}
> diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
> index b0879bd87e..fcab5a0904 100644
> --- a/nptl/nptl-init.c
> +++ b/nptl/nptl-init.c
> @@ -191,8 +191,6 @@ __pthread_initialize_minimal_internal (void)
> GL(dl_make_stack_executable_hook) = &__make_stacks_executable;
> #endif
>
> - GL(dl_init_static_tls) = &__pthread_init_static_tls;
> -
> /* Register the fork generation counter with the libc. */
> #ifndef TLS_MULTIPLE_THREADS_IN_TCB
> __libc_multiple_threads_ptr =
> diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
> index 00d2cfe764..0ce63672c4 100644
> --- a/nptl/pthreadP.h
> +++ b/nptl/pthreadP.h
> @@ -381,8 +381,6 @@ extern int __pthread_multiple_threads attribute_hidden;
> extern int *__libc_multiple_threads_ptr attribute_hidden;
> #endif
>
> -extern void __pthread_init_static_tls (struct link_map *) attribute_hidden;
> -
> extern size_t __pthread_get_minstack (const pthread_attr_t *attr);
>
> /* Namespace save aliases. */
> diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
> index 67c6686015..1b064c5894 100644
> --- a/sysdeps/generic/ldsodefs.h
> +++ b/sysdeps/generic/ldsodefs.h
> @@ -464,7 +464,9 @@ struct rtld_global
> /* Generation counter for the dtv. */
> EXTERN size_t _dl_tls_generation;
>
> +#if !THREAD_GSCOPE_IN_TCB
> EXTERN void (*_dl_init_static_tls) (struct link_map *);
> +#endif
>
> /* Scopes to free after next THREAD_GSCOPE_WAIT (). */
> EXTERN struct dl_scope_free_list
> @@ -1270,6 +1272,23 @@ extern void _dl_non_dynamic_init (void)
> extern void _dl_aux_init (ElfW(auxv_t) *av)
> attribute_hidden;
>
> +/* Initialize the static TLS space for the link map in all existing
> + threads. */
> +#if THREAD_GSCOPE_IN_TCB
> +void _dl_init_static_tls (struct link_map *map) attribute_hidden;
> +#endif
> +static inline void
> +dl_init_static_tls (struct link_map *map)
> +{
> +#if THREAD_GSCOPE_IN_TCB
> + /* The stack list is available to ld.so, so the initialization can
> + be handled within ld.so directly. */
> + _dl_init_static_tls (map);
> +#else
> + GL (dl_init_static_tls) (map);
> +#endif
> +}
> +
> /* Return true if the ld.so copy in this namespace is actually active
> and working. If false, the dl_open/dlfcn hooks have to be used to
> call into the outer dynamic linker (which happens after static
>
LGTM.
Thanks.
@@ -426,7 +426,7 @@ TLS generation counter wrapped! Please report this."));
_dl_update_slotinfo (imap->l_tls_modid);
#endif
- GL(dl_init_static_tls) (imap);
+ dl_init_static_tls (imap);
assert (imap->l_need_tls_init == 0);
}
}
@@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional)
(void) _dl_update_slotinfo (map->l_tls_modid);
#endif
- GL(dl_init_static_tls) (map);
+ dl_init_static_tls (map);
}
else
map->l_need_tls_init = 1;
@@ -141,6 +141,7 @@ cannot allocate memory in static TLS block"));
}
}
+#if !THREAD_GSCOPE_IN_TCB
/* Initialize static TLS area and DTV for current (only) thread.
libpthread implementations should provide their own hook
to handle all threads. */
@@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map)
memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
'\0', map->l_tls_blocksize - map->l_tls_initimage_size);
}
-
+#endif /* !THREAD_GSCOPE_IN_TCB */
void
_dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],
@@ -138,8 +138,6 @@ void *_dl_random;
#include <dl-procruntime.c>
#include <dl-procinfo.c>
-void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
-
size_t _dl_pagesize = EXEC_PAGESIZE;
size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ;
@@ -197,6 +195,7 @@ list_t _dl_stack_user;
int _dl_stack_cache_lock;
#else
int _dl_thread_gscope_count;
+void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
#endif
struct dl_scope_free_list *_dl_scope_free_list;
@@ -29,6 +29,10 @@
#include <dl-tls.h>
#include <ldsodefs.h>
+#if THREAD_GSCOPE_IN_TCB
+# include <list.h>
+#endif
+
#define TUNABLE_NAMESPACE rtld
#include <dl-tunables.h>
@@ -1005,3 +1009,38 @@ cannot create TLS data structures"));
listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
}
}
+
+#if THREAD_GSCOPE_IN_TCB
+static inline void __attribute__((always_inline))
+init_one_static_tls (struct pthread *curp, struct link_map *map)
+{
+# if TLS_TCB_AT_TP
+ void *dest = (char *) curp - map->l_tls_offset;
+# elif TLS_DTV_AT_TP
+ void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
+# else
+# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
+# endif
+
+ /* Initialize the memory. */
+ memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
+ '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
+}
+
+void
+_dl_init_static_tls (struct link_map *map)
+{
+ lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
+
+ /* Iterate over the list with system-allocated threads first. */
+ list_t *runp;
+ list_for_each (runp, &GL (dl_stack_used))
+ init_one_static_tls (list_entry (runp, struct pthread, list), map);
+
+ /* Now the list with threads using user-allocated stacks. */
+ list_for_each (runp, &GL (dl_stack_user))
+ init_one_static_tls (list_entry (runp, struct pthread, list), map);
+
+ lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
+}
+#endif /* THREAD_GSCOPE_IN_TCB */
@@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr,
struct dl_main_state state;
dl_main_state_init (&state);
+#if !THREAD_GSCOPE_IN_TCB
GL(dl_init_static_tls) = &_dl_nothread_init_static_tls;
+#endif
#if defined SHARED && defined _LIBC_REENTRANT \
&& defined __rtld_lock_default_lock_recursive
@@ -951,38 +951,3 @@ __reclaim_stacks (void)
GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
__default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
}
-
-
-static inline void __attribute__((always_inline))
-init_one_static_tls (struct pthread *curp, struct link_map *map)
-{
-# if TLS_TCB_AT_TP
- void *dest = (char *) curp - map->l_tls_offset;
-# elif TLS_DTV_AT_TP
- void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
-# else
-# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
-# endif
-
- /* Initialize the memory. */
- memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
- '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
-}
-
-void
-attribute_hidden
-__pthread_init_static_tls (struct link_map *map)
-{
- lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
-
- /* Iterate over the list with system-allocated threads first. */
- list_t *runp;
- list_for_each (runp, &GL (dl_stack_used))
- init_one_static_tls (list_entry (runp, struct pthread, list), map);
-
- /* Now the list with threads using user-allocated stacks. */
- list_for_each (runp, &GL (dl_stack_user))
- init_one_static_tls (list_entry (runp, struct pthread, list), map);
-
- lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
-}
@@ -191,8 +191,6 @@ __pthread_initialize_minimal_internal (void)
GL(dl_make_stack_executable_hook) = &__make_stacks_executable;
#endif
- GL(dl_init_static_tls) = &__pthread_init_static_tls;
-
/* Register the fork generation counter with the libc. */
#ifndef TLS_MULTIPLE_THREADS_IN_TCB
__libc_multiple_threads_ptr =
@@ -381,8 +381,6 @@ extern int __pthread_multiple_threads attribute_hidden;
extern int *__libc_multiple_threads_ptr attribute_hidden;
#endif
-extern void __pthread_init_static_tls (struct link_map *) attribute_hidden;
-
extern size_t __pthread_get_minstack (const pthread_attr_t *attr);
/* Namespace save aliases. */
@@ -464,7 +464,9 @@ struct rtld_global
/* Generation counter for the dtv. */
EXTERN size_t _dl_tls_generation;
+#if !THREAD_GSCOPE_IN_TCB
EXTERN void (*_dl_init_static_tls) (struct link_map *);
+#endif
/* Scopes to free after next THREAD_GSCOPE_WAIT (). */
EXTERN struct dl_scope_free_list
@@ -1270,6 +1272,23 @@ extern void _dl_non_dynamic_init (void)
extern void _dl_aux_init (ElfW(auxv_t) *av)
attribute_hidden;
+/* Initialize the static TLS space for the link map in all existing
+ threads. */
+#if THREAD_GSCOPE_IN_TCB
+void _dl_init_static_tls (struct link_map *map) attribute_hidden;
+#endif
+static inline void
+dl_init_static_tls (struct link_map *map)
+{
+#if THREAD_GSCOPE_IN_TCB
+ /* The stack list is available to ld.so, so the initialization can
+ be handled within ld.so directly. */
+ _dl_init_static_tls (map);
+#else
+ GL (dl_init_static_tls) (map);
+#endif
+}
+
/* Return true if the ld.so copy in this namespace is actually active
and working. If false, the dl_open/dlfcn hooks have to be used to
call into the outer dynamic linker (which happens after static