[RFC,22/22] Add Infinity notes implementing td_ta_map_lwp2thr
Commit Message
This commit adds the source code for the two Infinity notes that
implement libthread_db's td_ta_map_lwp2thr function. It also adds
an I8X testcase for it, and hooks it into "make check".
The build rules are conditional in that if I8C is not installed then
no notes will be built; the resulting libpthread.so will be as before.
If I8C is installed but I8X is not then the notes will be built but
untested. This combination is unlikely in practise as I8C and I8X are
part of the same package.
The source code is split into five files:
infinity-nptl.i8 contains some common definitions and will
be included by all other infinity-*.i8 files under nptl.
infinity-map_lwp2thr.i8 contains "libpthread::map_lwp2thr",
the note function that debuggers etc will call.
Libraries that contain Infinity notes should provide
documentation on the API of those notes. For this note
this is basically that it takes one argument, an integer
specifying the LWPID of interest, and returns two results,
an integer status code (one of libthread_db's: TD_OK, TD_ERR,
etc) and a pointer which is the result of the call iff the
returned status was TD_OK. The returned pointer is undefined
if the returned status was not TD_OK.
A further part of libpthread::map_lwp2thr's API is that it
requires two external functions: procservice::get_register
and procservice::get_thread_area.
(External functions must be defined *somehow* before this note may
be executed, in much the same way that programs wishing to use
libthread_db's td_ta_map_lwp2thr function must implement the
proc_service functions it uses. They could be Infinity notes
from some other library, but in this case it's more likely that
the program executing the note would provide them directly.)
infinity-lookup_th_unique_{cta,reg,rta}.i8 contain
"libpthread::__lookup_th_unique" for constant thread area,
register, and register thread area platforms respectively. This
function is used by map_lwp2thr. Which file is built depends on
macros defined in the platform's tls.h. The double-underscore
preceeding its name indicates to note consumers that this function
is not part of libpthread's exported Infinity API. It will be
part of the spec that consumers (i.e. programs that execute notes)
must not call double-underscore functions directly.
---
nptl/Makefile | 26 ++++++
nptl/infinity-lookup_th_unique_cta.i8 | 61 +++++++++++++
nptl/infinity-lookup_th_unique_reg.i8 | 76 ++++++++++++++++
nptl/infinity-lookup_th_unique_rta.i8 | 140 +++++++++++++++++++++++++++++
nptl/infinity-map_lwp2thr.i8 | 121 +++++++++++++++++++++++++
nptl/infinity-nptl.i8 | 36 ++++++++
nptl/tst-infinity-map_lwp2thr.py | 155 +++++++++++++++++++++++++++++++++
7 files changed, 615 insertions(+), 0 deletions(-)
create mode 100644 nptl/infinity-lookup_th_unique_cta.i8
create mode 100644 nptl/infinity-lookup_th_unique_reg.i8
create mode 100644 nptl/infinity-lookup_th_unique_rta.i8
create mode 100644 nptl/infinity-map_lwp2thr.i8
create mode 100644 nptl/infinity-nptl.i8
create mode 100644 nptl/tst-infinity-map_lwp2thr.py
Comments
I like the idea of improving debugging capabilities with something new
than libpthread_db, however I do not see the best approach of using
the i8 files directly in GLIBC. If I understood correctly from the
link you posted with this documentation, it is still a very WIP project
with rough places and the language/compiler is still being constantly
updated.
Also I do not like to add another tool dependency for complete GLIBC
build and one in such state (GLIBC do use python for benchmark but
it is not really required for build afaik). So which is the issue
of shipping the ELF notes in assembly format instead?
On 20-10-2015 12:56, Gary Benson wrote:
> This commit adds the source code for the two Infinity notes that
> implement libthread_db's td_ta_map_lwp2thr function. It also adds
> an I8X testcase for it, and hooks it into "make check".
>
> The build rules are conditional in that if I8C is not installed then
> no notes will be built; the resulting libpthread.so will be as before.
> If I8C is installed but I8X is not then the notes will be built but
> untested. This combination is unlikely in practise as I8C and I8X are
> part of the same package.
>
> The source code is split into five files:
>
> infinity-nptl.i8 contains some common definitions and will
> be included by all other infinity-*.i8 files under nptl.
>
> infinity-map_lwp2thr.i8 contains "libpthread::map_lwp2thr",
> the note function that debuggers etc will call.
>
> Libraries that contain Infinity notes should provide
> documentation on the API of those notes. For this note
> this is basically that it takes one argument, an integer
> specifying the LWPID of interest, and returns two results,
> an integer status code (one of libthread_db's: TD_OK, TD_ERR,
> etc) and a pointer which is the result of the call iff the
> returned status was TD_OK. The returned pointer is undefined
> if the returned status was not TD_OK.
>
> A further part of libpthread::map_lwp2thr's API is that it
> requires two external functions: procservice::get_register
> and procservice::get_thread_area.
>
> (External functions must be defined *somehow* before this note may
> be executed, in much the same way that programs wishing to use
> libthread_db's td_ta_map_lwp2thr function must implement the
> proc_service functions it uses. They could be Infinity notes
> from some other library, but in this case it's more likely that
> the program executing the note would provide them directly.)
>
> infinity-lookup_th_unique_{cta,reg,rta}.i8 contain
> "libpthread::__lookup_th_unique" for constant thread area,
> register, and register thread area platforms respectively. This
> function is used by map_lwp2thr. Which file is built depends on
> macros defined in the platform's tls.h. The double-underscore
> preceeding its name indicates to note consumers that this function
> is not part of libpthread's exported Infinity API. It will be
> part of the spec that consumers (i.e. programs that execute notes)
> must not call double-underscore functions directly.
> ---
> nptl/Makefile | 26 ++++++
> nptl/infinity-lookup_th_unique_cta.i8 | 61 +++++++++++++
> nptl/infinity-lookup_th_unique_reg.i8 | 76 ++++++++++++++++
> nptl/infinity-lookup_th_unique_rta.i8 | 140 +++++++++++++++++++++++++++++
> nptl/infinity-map_lwp2thr.i8 | 121 +++++++++++++++++++++++++
> nptl/infinity-nptl.i8 | 36 ++++++++
> nptl/tst-infinity-map_lwp2thr.py | 155 +++++++++++++++++++++++++++++++++
> 7 files changed, 615 insertions(+), 0 deletions(-)
> create mode 100644 nptl/infinity-lookup_th_unique_cta.i8
> create mode 100644 nptl/infinity-lookup_th_unique_reg.i8
> create mode 100644 nptl/infinity-lookup_th_unique_rta.i8
> create mode 100644 nptl/infinity-map_lwp2thr.i8
> create mode 100644 nptl/infinity-nptl.i8
> create mode 100644 nptl/tst-infinity-map_lwp2thr.py
>
> diff --git a/nptl/Makefile b/nptl/Makefile
> index 0272813..eb90e66 100644
> --- a/nptl/Makefile
> +++ b/nptl/Makefile
> @@ -138,6 +138,12 @@ libpthread-routines = nptl-init vars events version pt-interp \
> # pthread_setgid pthread_setegid pthread_setregid \
> # pthread_setresgid
>
> +ifneq ($(I8C),no)
> +infinity-routines = infinity-map_lwp2thr
> +
> +libpthread-routines += $(infinity-routines)
> +endif
> +
> libpthread-shared-only-routines = version pt-interp pt-allocrtsig \
> unwind-forcedunwind
> libpthread-static-only-routines = pthread_atfork
> @@ -414,6 +420,26 @@ ifneq ($(have-cxx-thread_local),yes)
> tests-unsupported += tst-thread_local1
> endif
>
> +tests += tst-infinity
> +ifneq ($(I8C),no)
> +ifneq ($(I8X),no)
> +infinity-tests = $(addprefix tst-,$(addsuffix .py,$(infinity-routines)))
> +
> +$(objpfx)tst-infinity.out: $(objpfx)libpthread.so$(libpthread.so-version) \
> + $(infinity-tests)
> + $(I8X) -i $< -I$(common-objpfx) $(infinity-tests) > $@; \
> + $(evaluate-test)
> +
> +tests-special += $(objpfx)tst-infinity.out
> +else
> +# I8X is not installed; the tests cannot be run.
> +tests-unsupported += tst-infinity
> +endif
> +else
> +# I8C is not installed; libpthread.so will not contain notes.
> +tests-unsupported += tst-infinity
> +endif
> +
> include ../Rules
>
> ifeq (yes,$(build-shared))
> diff --git a/nptl/infinity-lookup_th_unique_cta.i8 b/nptl/infinity-lookup_th_unique_cta.i8
> new file mode 100644
> index 0000000..0af457e
> --- /dev/null
> +++ b/nptl/infinity-lookup_th_unique_cta.i8
> @@ -0,0 +1,61 @@
> +/* Which thread is running on an LWP?
> + Copyright (C) 2003-2015 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +
> +/* Given an lwpid_t identifying an LWP, return TD_OK and the
> + descriptor of the thread running on it, or a non-TD_OK
> + td_err_e code indicating the reason for failure and an
> + undefined value that must be ignored. The caller must
> + ensure that __pthread_initialize_minimal has gotten far
> + enough; see the comments in infinity_map_lwp2thr.i8 for
> + specifics. */
> +
> +define libpthread::__lookup_th_unique returns td_err_e, ptr
> + argument lwpid_t lwpid
> + extern ps_get_ta_f procservice::get_thread_area
> +
> + // stack 0: procservice::get_thread_area
> + // stack 1: lwpid
> + load I8_TS_CTA_VALUE
> + // stack 0: I8_TS_CTA_VALUE
> + // stack 1: procservice::get_thread_area
> + // stack 2: lwpid
> + swap
> + // stack 0: procservice::get_thread_area
> + // stack 1: I8_TS_CTA_VALUE
> + // stack 2: lwpid
> + call
> + // stack 0: PS_OK (hopefully) or something else
> + // stack 1: the thing we want (hopefully) or junk
> + load PS_OK
> + // stack 0: PS_OK
> + // stack 1: PS_OK (hopefully) or something else
> + // stack 2: the thing we want (hopefully) or junk
> + bne get_thread_area_failed
> + // stack 0: the thing we want
> + load TD_OK
> + // stack 0: TD_OK
> + // stack 1: the thing we want
> + return
> +
> +get_thread_area_failed:
> + // stack 0: junk
> + load TD_ERR
> + // stack 0: TD_OK
> + // stack 1: junk
> + return
> diff --git a/nptl/infinity-lookup_th_unique_reg.i8 b/nptl/infinity-lookup_th_unique_reg.i8
> new file mode 100644
> index 0000000..07aa3a0
> --- /dev/null
> +++ b/nptl/infinity-lookup_th_unique_reg.i8
> @@ -0,0 +1,76 @@
> +/* Which thread is running on an LWP?
> + Copyright (C) 2003-2015 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +
> +/* Given an lwpid_t identifying an LWP, return TD_OK and the
> + descriptor of the thread running on it, or a non-TD_OK
> + td_err_e code indicating the reason for failure and an
> + undefined value that must be ignored. The caller must
> + ensure that __pthread_initialize_minimal has gotten far
> + enough; see the comments in infinity_map_lwp2thr.i8 for
> + specifics. */
> +
> +define libpthread::__lookup_th_unique returns td_err_e, ptr
> + argument lwpid_t lwpid
> + extern ps_getreg_f procservice::get_register
> +
> + // stack 0: procservice::get_register
> + // stack 1: lwpid
> + load I8_TS_REG_OFFSET
> + // stack 0: I8_TS_REG_OFFSET
> + // stack 1: procservice::get_thread_area
> + // stack 2: lwpid
> + swap
> + // stack 0: procservice::get_thread_area
> + // stack 1: I8_TS_REG_OFFSET
> + // stack 2: lwpid
> + load I8_TS_REG_SIZE
> + // stack 0: I8_TS_REG_SIZE
> + // stack 1: procservice::get_thread_area
> + // stack 2: I8_TS_REG_OFFSET
> + // stack 3: lwpid
> + swap
> + // stack 1: procservice::get_thread_area
> + // stack 0: I8_TS_REG_SIZE
> + // stack 2: I8_TS_REG_OFFSET
> + // stack 3: lwpid
> + call
> + // stack 0: PS_OK (hopefully) or something else
> + // stack 1: contents of register (hopefully) or junk
> + load PS_OK
> + // stack 0: PS_OK
> + // stack 1: PS_OK (hopefully) or something else
> + // stack 2: contents of register (hopefully) or junk
> + bne get_register_failed
> + // stack 0: contents of register
> + load I8_TS_REG_BIAS
> + // stack 0: I8_TS_REG_BIAS
> + // stack 1: contents of register
> + add
> + // stack 0: biased contents of register
> + load TD_OK
> + // stack 0: TD_OK
> + // stack 1: biased contents of register
> + return
> +
> +get_register_failed:
> + // stack 0: junk
> + load TD_ERR
> + // stack 0: TD_OK
> + // stack 1: junk
> + return
> diff --git a/nptl/infinity-lookup_th_unique_rta.i8 b/nptl/infinity-lookup_th_unique_rta.i8
> new file mode 100644
> index 0000000..d892566
> --- /dev/null
> +++ b/nptl/infinity-lookup_th_unique_rta.i8
> @@ -0,0 +1,140 @@
> +/* Which thread is running on an LWP?
> + Copyright (C) 2003-2015 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +
> +/* Given an lwpid_t identifying an LWP, return TD_OK and the
> + descriptor of the thread running on it, or a non-TD_OK
> + td_err_e code indicating the reason for failure and an
> + undefined value that must be ignored. The caller must
> + ensure that __pthread_initialize_minimal has gotten far
> + enough; see the comments in infinity_map_lwp2thr.i8 for
> + specifics. */
> +
> +define libpthread::__lookup_th_unique returns td_err_e, ptr
> + argument lwpid_t lwpid
> + extern ps_getreg_f procservice::get_register
> + extern ps_get_ta_f procservice::get_thread_area
> +
> + // stack 0: procservice::get_thread_area
> + // stack 1: procservice::get_reg
> + // stack 2: lwpid
> + load lwpid
> + // stack 0: lwpid
> + // stack 1: procservice::get_thread_area
> + // stack 2: procservice::get_reg
> + // stack 3: lwpid
> + rot
> + // stack 0: procservice::get_thread_area
> + // stack 1: procservice::get_reg
> + // stack 2: lwpid
> + // stack 3: lwpid
> + rot
> + // stack 0: procservice::get_reg
> + // stack 1: lwpid
> + // stack 2: procservice::get_thread_area
> + // stack 3: lwpid
> + load I8_TS_RTA_OFFSET
> + // stack 0: I8_TS_RTA_OFFSET
> + // stack 1: procservice::get_thread_area
> + // stack 2: lwpid
> + // stack 3: procservice::get_thread_area
> + // stack 4: lwpid
> + swap
> + // stack 0: procservice::get_thread_area
> + // stack 1: I8_TS_RTA_OFFSET
> + // stack 2: lwpid
> + // stack 3: procservice::get_thread_area
> + // stack 4: lwpid
> + load I8_TS_RTA_SIZE
> + // stack 0: I8_TS_RTA_SIZE
> + // stack 1: procservice::get_thread_area
> + // stack 2: I8_TS_RTA_OFFSET
> + // stack 3: lwpid
> + // stack 4: procservice::get_thread_area
> + // stack 5: lwpid
> + swap
> + // stack 0: procservice::get_thread_area
> + // stack 1: I8_TS_RTA_SIZE
> + // stack 2: I8_TS_RTA_OFFSET
> + // stack 3: lwpid
> + // stack 4: procservice::get_thread_area
> + // stack 5: lwpid
> + call
> + // stack 0: PS_OK (hopefully) or something else
> + // stack 1: contents of register (hopefully) or junk
> + // stack 2: procservice::get_thread_area
> + // stack 3: lwpid
> + load PS_OK
> + // stack 0: PS_OK
> + // stack 1: PS_OK (hopefully) or something else
> + // stack 2: contents of register (hopefully) or junk
> + // stack 3: procservice::get_thread_area
> + // stack 4: lwpid
> + bne get_register_failed
> + // stack 0: contents of register
> + // stack 1: procservice::get_thread_area
> + // stack 2: lwpid
> + cast 0 int
> + // stack 0: I8_TS_RTA_SCALE
> + // stack 1: contents of register
> + // stack 2: procservice::get_thread_area
> + // stack 3: lwpid
> + load I8_TS_RTA_SCALE
> + // stack 0: I8_TS_RTA_SCALE
> + // stack 1: contents of register
> + // stack 2: procservice::get_thread_area
> + // stack 3: lwpid
> + shr
> + // stack 0: scaled contents of register
> + // stack 1: procservice::get_thread_area
> + // stack 2: lwpid
> + swap
> + // stack 0: procservice::get_thread_area
> + // stack 1: scaled contents of register
> + // stack 2: lwpid
> + call
> + // stack 0: PS_OK (hopefully) or something else
> + // stack 1: the thing we want (hopefully) or junk
> + load PS_OK
> + // stack 0: PS_OK
> + // stack 1: PS_OK (hopefully) or something else
> + // stack 2: the thing we want (hopefully) or junk
> + bne get_thread_area_failed
> + // stack 0: the thing we want
> + load TD_OK
> + // stack 0: TD_OK
> + // stack 1: the thing we want
> + return
> +
> +get_register_failed:
> + // stack 0: junk
> + // stack 1: procservice::get_thread_area
> + // stack 2: lwpid
> + load TD_ERR
> + // stack 0: TD_OK
> + // stack 1: junk
> + // stack 2: procservice::get_thread_area
> + // stack 3: lwpid
> + return
> +
> +get_thread_area_failed:
> + // stack 0: junk
> + load TD_ERR
> + // stack 0: TD_ERR
> + // stack 1: junk
> + return
> diff --git a/nptl/infinity-map_lwp2thr.i8 b/nptl/infinity-map_lwp2thr.i8
> new file mode 100644
> index 0000000..51b21ea
> --- /dev/null
> +++ b/nptl/infinity-map_lwp2thr.i8
> @@ -0,0 +1,121 @@
> +/* Which thread is running on an LWP?
> + Copyright (C) 2003-2015 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include "infinity-nptl.i8"
> +#include "infinity-thread_self.h"
> +
> +#if I8_THREAD_SELF == I8_TS_CONST_THREAD_AREA
> +# include "infinity-lookup_th_unique_cta.i8"
> +#endif
> +#if I8_THREAD_SELF == I8_TS_REGISTER
> +# include "infinity-lookup_th_unique_reg.i8"
> +#endif
> +#if I8_THREAD_SELF == I8_TS_REGISTER_THREAD_AREA
> +# include "infinity-lookup_th_unique_rta.i8"
> +#endif
> +
> +/* Given an lwpid_t identifying an LWP, return TD_OK and the
> + descriptor of the thread running on it, or a non-TD_OK
> + td_err_e code indicating the reason for failure and an
> + undefined value that must be ignored. Thread descriptors
> + are opaque pointers and should not be dereferenced outside
> + of this library. */
> +
> +define libpthread::map_lwp2thr returns td_err_e, ptr
> + argument lwpid_t lwpid
> + extern func td_err_e, ptr (lwpid_t) __lookup_th_unique
> + extern func pid_t () i8core::getpid
> + extern ptr __stack_user
> +
> + /* We cannot rely on thread registers and such information at all
> + before __pthread_initialize_minimal has gotten far enough: they
> + sometimes contain garbage left by the kernel at exec that would
> + confuse us. If it looks like initialization is incomplete we
> + fake a special descriptor of NULL to indicate the initial thread.
> + Other routines in this library recognise this special descriptor
> + and act accordingly. */
> +
> + // stack 0: __stack_user
> + // stack 1: i8core::getpid
> + // stack 2: __lookup_th_unique
> + // stack 3: lwpid
> + load LIST_T_NEXT_OFFSET
> + // stack 0: LIST_T_NEXT_OFFSET
> + // stack 1: __stack_user
> + // stack 2: i8core::getpid
> + // stack 3: __lookup_th_unique
> + // stack 4: lwpid
> + add
> + // stack 0: __stack_user + LIST_T_NEXT_OFFSET
> + // stack 1: i8core::getpid
> + // stack 2: __lookup_th_unique
> + // stack 3: lwpid
> + deref ptr
> + // stack 0: __stack_user->next
> + // stack 1: i8core::getpid
> + // stack 2: __lookup_th_unique
> + // stack 3: lwpid
> + load NULL
> + bne libpthread_is_initialized
> +
> +initialization_is_incomplete:
> + // stack 0: i8core::getpid
> + // stack 1: __lookup_th_unique
> + // stack 2: lwpid
> + swap
> + // stack 0: __lookup_th_unique
> + // stack 1: i8core::getpid
> + // stack 2: lwpid
> + drop
> + // stack 0: i8core::getpid
> + // stack 1: lwpid
> + load NULL
> + // stack 0: NULL
> + // stack 1: i8core::getpid
> + // stack 2: lwpid
> + rot
> + // stack 0: i8core::getpid
> + // stack 1: lwpid
> + // stack 2: NULL
> + call
> + // stack 0: main pid
> + // stack 1: lwpid
> + // stack 2: NULL
> + beq is_main_thread
> +
> +not_main_thread:
> + // stack 0: NULL
> + load TD_ERR
> + return
> +
> +is_main_thread:
> + // stack 0: NULL
> + load TD_OK
> + return
> +
> +libpthread_is_initialized:
> + // stack 0: i8core::getpid
> + // stack 1: __lookup_th_unique
> + // stack 2: lwpid
> + drop
> + // stack 0: __lookup_th_unique
> + // stack 1: lwpid
> + call
> + // stack 0: TD_OK (or not)
> + // stack 1: descriptor (or junk)
> + return
> diff --git a/nptl/infinity-nptl.i8 b/nptl/infinity-nptl.i8
> new file mode 100644
> index 0000000..62c6012
> --- /dev/null
> +++ b/nptl/infinity-nptl.i8
> @@ -0,0 +1,36 @@
> +/* Common definitions for NPTL Infinity functions.
> + Copyright (C) 2015 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +/* libpthread and libthread_db generated constants. */
> +#include "infinity-nptl-constants.h"
> +#include "infinity-nptl_db-constants.h"
> +
> +/* XXX. */
> +typedef int size_t
> +
> +/* XXX. */
> +typedef s32 pid_t
> +typedef pid_t lwpid_t
> +
> +/* XXX. */
> +typedef s32 td_err_e
> +typedef s32 ps_err_e
> +
> +/* XXX. */
> +typedef func ps_err_e, ptr (lwpid_t, int, int) ps_getreg_f
> +typedef func ps_err_e, ptr (lwpid_t, int) ps_get_ta_f
> diff --git a/nptl/tst-infinity-map_lwp2thr.py b/nptl/tst-infinity-map_lwp2thr.py
> new file mode 100644
> index 0000000..6c7e37e
> --- /dev/null
> +++ b/nptl/tst-infinity-map_lwp2thr.py
> @@ -0,0 +1,155 @@
> +from i8c.runtime import TestCase
> +import struct
> +
> +TestCase.import_builtin_constants()
> +TestCase.import_constants_from("infinity-nptl-constants.h")
> +TestCase.import_constants_from("infinity-nptl_db-constants.h")
> +
> +class TestMapLwp2Thr(TestCase):
> + TESTFUNC = "libpthread::map_lwp2thr(i)ip"
> + MAIN_PID = 30000
> +
> + def setUp(self):
> + # Create flags
> + self.ps_get_register_called = False
> + self.ps_get_thread_area_called = False
> + # Store the address of __stack_user
> + note = self.i8ctx.get_function(self.TESTFUNC)
> + symbols = note.external_pointers
> + self.assertEqual(len(symbols), 1)
> + self.stack_user_p = symbols[0]
> +
> + def read_memory(self, fmt, addr):
> + # The only dereference we do is __stack_user.next
> + self.assertEqual(addr, self.stack_user_p + LIST_T_NEXT_OFFSET)
> + return struct.pack(fmt, self.STACK_USER_NEXT)
> +
> + def call_i8core_getpid(self):
> + """Implementation of i8core::getpid."""
> + return self.MAIN_PID
> +
> + def call_procservice_get_register(self, lwpid, offset, size):
> + """Implementation of procservice::get_register."""
> + self.assertFalse(self.ps_get_register_called)
> + result = getattr(self, "PS_GETREG_RESULT", None)
> + if result is None:
> + self.fail("unexpected ps_get_register")
> + self.assertEqual(lwpid, self.lwpid)
> + self.assertNotEqual(offset, self.lwpid)
> + self.assertGreaterEqual(offset, 0)
> + # We can't really say much about offset. It's an offset into
> + # a prgregset_t structure, so it's probably not huge and it's
> + # probably aligned to the machine's wordsize.
> + self.assertLess(offset, 128 * 8) # =128 64-bit registers (IA-64)
> + bytes_per_word, check = divmod(self.i8ctx.wordsize, 8)
> + self.assertNotEqual(bytes_per_word, 0)
> + self.assertEqual(check, 0)
> + self.assertEqual(offset % bytes_per_word, 0)
> + self.assertIn(size, (8, 16, 32, 64))
> + self.assertLessEqual(size, self.i8ctx.wordsize)
> + self.ps_get_register_called = True
> + return result
> +
> + def call_procservice_get_thread_area(self, lwpid, idx):
> + """Implementation of procservice::get_thread_area."""
> + self.assertFalse(self.ps_get_thread_area_called)
> + result = getattr(self, "PS_GET_TA_RESULT", None)
> + if result is None:
> + self.fail("unexpected ps_get_thread_area")
> + self.assertEqual(lwpid, self.lwpid)
> + self.assertNotEqual(idx, self.lwpid)
> + self.ps_get_thread_area_called = True
> + return result
> +
> + def check_I8_TS_CONST_THREAD_AREA_result(self, result):
> + # The result is whatever ps_get_thread_area returned
> + self.assertTrue(self.ps_get_thread_area_called)
> + self.assertEqual(result[0], TD_OK)
> + self.assertNotEqual(result[1], 0)
> + self.assertEqual(result[1], self.PS_GET_TA_RESULT[1])
> +
> + def check_I8_TS_REGISTER_result(self, result):
> + # The result is what ps_get_register returned with some
> + # bias added. We'll assume the bias is fairly small.
> + self.assertTrue(self.ps_get_register_called)
> + self.assertEqual(result[0], TD_OK)
> + self.assertNotEqual(result[1], 0)
> + bias = result[1] - self.PS_GETREG_RESULT[1]
> + self.assertLess(abs(bias), 16384)
> +
> + def check_I8_TS_REGISTER_THREAD_AREA_result(self, result):
> + # The result is whatever ps_get_thread_area returned
> + self.assertTrue(self.ps_get_register_called)
> + self.assertTrue(self.ps_get_thread_area_called)
> + self.assertEqual(result[0], TD_OK)
> + self.assertNotEqual(result[1], 0)
> + self.assertEqual(result[1], self.PS_GET_TA_RESULT[1])
> +
> +class TestMapLwp2Thr_uninit(TestMapLwp2Thr):
> + STACK_USER_NEXT = NULL
> +
> + def test_map_lwp2thr(self):
> + """map_lwp2thr (nptl uninitialized, lwpid == main PID)"""
> + result = self.i8ctx.call(self.TESTFUNC, self.MAIN_PID)
> + self.assertEqual(len(result), 2)
> + self.assertEqual(result[0], TD_OK)
> + self.assertEqual(result[1], NULL)
> +
> +class TestMapLwp2Thr_uninit_wrongpid(TestMapLwp2Thr):
> + STACK_USER_NEXT = NULL
> +
> + def test_map_lwp2thr(self):
> + """map_lwp2thr (nptl uninitialized, lwpid != main PID)"""
> + result = self.i8ctx.call(self.TESTFUNC, self.MAIN_PID + 1)
> + self.assertEqual(len(result), 2)
> + self.assertEqual(result[0], TD_ERR)
> +
> +class TestMapLwp2Thr_init_getreg_fail(TestMapLwp2Thr):
> + STACK_USER_NEXT = 0x1fff
> + PS_GETREG_RESULT = PS_ERR, 0x23ff00fa
> + PS_GET_TA_RESULT = PS_OK, 0x89ab1234
> +
> + def test_map_lwp2thr(self):
> + """map_lwp2thr (nptl initialized, ps_get_register fails)"""
> + self.lwpid = self.MAIN_PID + 1
> + result = self.i8ctx.call(self.TESTFUNC, self.lwpid)
> + self.assertEqual(len(result), 2)
> + if self.ps_get_register_called:
> + self.assertEqual(result[0], TD_ERR)
> + else:
> + # This failure isn't a problem for this platform
> + self.check_I8_TS_CONST_THREAD_AREA_result(result)
> +
> +class TestMapLwp2Thr_init_gta_fail(TestMapLwp2Thr):
> + STACK_USER_NEXT = 0x1fff
> + PS_GETREG_RESULT = PS_OK, 0x23ff00fa
> + PS_GET_TA_RESULT = PS_ERR, 0x89ab1234
> +
> + def test_map_lwp2thr(self):
> + """map_lwp2thr (nptl initialized, ps_get_thread_area fails)"""
> + self.lwpid = self.MAIN_PID + 1
> + result = self.i8ctx.call(self.TESTFUNC, self.lwpid)
> + self.assertEqual(len(result), 2)
> + if self.ps_get_thread_area_called:
> + self.assertEqual(result[0], TD_ERR)
> + else:
> + # This failure isn't a problem for this platform
> + self.check_I8_TS_REGISTER_result(result)
> +
> +class TestMapLwp2Thr_init_gta_ok(TestMapLwp2Thr):
> + STACK_USER_NEXT = 0x1fff
> + PS_GETREG_RESULT = PS_OK, 0x23ff00fa
> + PS_GET_TA_RESULT = PS_OK, 0x89ab1234
> +
> + def test_map_lwp2thr(self):
> + """map_lwp2thr (nptl initialized, everything worked)"""
> + self.lwpid = self.MAIN_PID + 1
> + result = self.i8ctx.call(self.TESTFUNC, self.lwpid)
> + self.assertEqual(len(result), 2)
> + if self.ps_get_thread_area_called:
> + if self.ps_get_register_called:
> + self.check_I8_TS_REGISTER_THREAD_AREA_result(result)
> + else:
> + self.check_I8_TS_CONST_THREAD_AREA_result(result)
> + else:
> + self.check_I8_TS_REGISTER_result(result)
>
Adhemerval Zanella wrote:
> I like the idea of improving debugging capabilities with something
> new than libpthread_db, however I do not see the best approach of
> using the i8 files directly in GLIBC. If I understood correctly from
> the link you posted with this documentation, it is still a very WIP
> project with rough places and the language/compiler is still being
> constantly updated.
It is a work-in-progress right now. It's being developed to allow GDB
to debug multithreaded inferiors, so this patch series (or, rather, the
complete version that will follow in the next few months) is the work
that will turn Infinity from WIP to something complete.
I mailed this series as an RFA to get some early feedback. I'm
anticipating that a prerequisite of a GLIBC commit will be a GDB that
fully supports Infinity (and a prerequisite of a GDB commit will be a
GLIBC with Infinity notes) and what I *didn't* want to do was appear
out of the blue with a complete system I'd spent six months writing
away by myself only for somebody to point out a killer flaw that meant
the whole thing was junk. Also because I recognise that input from
the wider community will make the final result something way better
than I could achieve on my own.
So yes, it's very work-in-progress right now, but by the time it's
ready to commit it should be pretty well baked and polished.
> Also I do not like to add another tool dependency for complete GLIBC
> build and one in such state (GLIBC do use python for benchmark but
> it is not really required for build afaik). So which is the issue of
> shipping the ELF notes in assembly format instead?
The initial reason I opted for a compiled source approach is that the
note format is somewhat complex. The format is documented here:
https://sourceware.org/gdb/wiki/NoteFormat
I want Infinity to be future-proof. Now that libthread_db exists,
GLIBC is essentially bound to support it *forever*. I don't want to
introduce another debugger interface that we'll have to support
forever unless I can be fairly confident we won't have to replace that
five or ten years down the line. So there's a lot of requirements in
the note format specification that (should!) mean that any new feature
we require can be added without breaking existing note consumers.
The second reason for using a compiler (or, more generally, tools) is
testing. We could ship the notes in assembly language but then how do
you test them? I don't like the idea of GLIBC shipping with untested
code, especially since the result of a breakage might not be
immediately obvious: if we ship a broken note in a GLIBC release it
might well not be noticed until it makes its way into some distro and
some user finds they can't debug their application because one of the
thread-debugging notes is invalid. That's why the required tools are
a compiler *and* a unit tester. I don't know if you saw, but the
testcase for the note in this patch exercises every path through the
compiled note. It's not perfect, but it's as close as I can get. So,
each note I add to GLIBC will have an accompanying testcase so I can
be as certain as I can be that if "make check" passed then the notes
in the files will work.
For me, the second reason (testing) is the more compelling of the two.
And if you're going to require an external tool for testing, why not
require an external compiler too?
I appreciate that people don't want to add more build dependencies to
GLIBC but I don't really have a way around that. To mitigate the
impact I've written the tools without using any external libraries,
and I've written them to work on as wide a range of Python versions
as is possible. It's not possible to write code that works both
for Python < 2.6 and for Python >= 3.0 simultaneously so 2.6 is the
oldest you can use.
As a final (not entirely related) point, while I'm talking about
testing, I'd like to note that while the compiler and tester are both
works-in-progress, they both have pretty comprehensive test suites.
The compiler has tests with 98% coverage so I'm pretty sure it's doing
what I think it is on all supported versions of Python. I'm using
Travis CI to test every commit I make with seven different versions of
Python from 2.7 to the latest nightly:
https://travis-ci.org/gbenson/i8c
The tester has only 87% coverage but I'm working on that :)
Cheers,
Gary
On 21-10-2015 12:42, Gary Benson wrote:
> Adhemerval Zanella wrote:
>> I like the idea of improving debugging capabilities with something
>> new than libpthread_db, however I do not see the best approach of
>> using the i8 files directly in GLIBC. If I understood correctly from
>> the link you posted with this documentation, it is still a very WIP
>> project with rough places and the language/compiler is still being
>> constantly updated.
>
> It is a work-in-progress right now. It's being developed to allow GDB
> to debug multithreaded inferiors, so this patch series (or, rather, the
> complete version that will follow in the next few months) is the work
> that will turn Infinity from WIP to something complete.
>
> I mailed this series as an RFA to get some early feedback. I'm
> anticipating that a prerequisite of a GLIBC commit will be a GDB that
> fully supports Infinity (and a prerequisite of a GDB commit will be a
> GLIBC with Infinity notes) and what I *didn't* want to do was appear
> out of the blue with a complete system I'd spent six months writing
> away by myself only for somebody to point out a killer flaw that meant
> the whole thing was junk. Also because I recognise that input from
> the wider community will make the final result something way better
> than I could achieve on my own.
>
> So yes, it's very work-in-progress right now, but by the time it's
> ready to commit it should be pretty well baked and polished.
I did say it was flawed or even half-baked, my concern is due it uses
a non well establish language it may add more maintaining burden than
use assembly drop releases. What happens if had or decided to extend
I8X to cover a new feature and add this dependency on the GLIBC code?
Will you always provide backwards compatibility? Will we need to tie
GLIBC releases to some I8X release or will be feature based?
>
>> Also I do not like to add another tool dependency for complete GLIBC
>> build and one in such state (GLIBC do use python for benchmark but
>> it is not really required for build afaik). So which is the issue of
>> shipping the ELF notes in assembly format instead?
>
> The initial reason I opted for a compiled source approach is that the
> note format is somewhat complex. The format is documented here:
>
> https://sourceware.org/gdb/wiki/NoteFormat
>
> I want Infinity to be future-proof. Now that libthread_db exists,
> GLIBC is essentially bound to support it *forever*. I don't want to
> introduce another debugger interface that we'll have to support
> forever unless I can be fairly confident we won't have to replace that
> five or ten years down the line. So there's a lot of requirements in
> the note format specification that (should!) mean that any new feature
> we require can be added without breaking existing note consumers.
>
> The second reason for using a compiler (or, more generally, tools) is
> testing. We could ship the notes in assembly language but then how do
> you test them? I don't like the idea of GLIBC shipping with untested
> code, especially since the result of a breakage might not be
> immediately obvious: if we ship a broken note in a GLIBC release it
> might well not be noticed until it makes its way into some distro and
> some user finds they can't debug their application because one of the
> thread-debugging notes is invalid. That's why the required tools are
> a compiler *and* a unit tester. I don't know if you saw, but the
> testcase for the note in this patch exercises every path through the
> compiled note. It's not perfect, but it's as close as I can get. So,
> each note I add to GLIBC will have an accompanying testcase so I can
> be as certain as I can be that if "make check" passed then the notes
> in the files will work.
>
> For me, the second reason (testing) is the more compelling of the two.
> And if you're going to require an external tool for testing, why not
> require an external compiler too?
AFAIK the only non-trivial build dependency for testing is perl and
I do not see a way forward to add another interpreter dependency. It
will be even harder on cross-compiling environment to do native testing
with such dependency.
>
> I appreciate that people don't want to add more build dependencies to
> GLIBC but I don't really have a way around that. To mitigate the
> impact I've written the tools without using any external libraries,
> and I've written them to work on as wide a range of Python versions
> as is possible. It's not possible to write code that works both
> for Python < 2.6 and for Python >= 3.0 simultaneously so 2.6 is the
> oldest you can use.
So the compelling reason I see so far to do not make assembly releases
and to not add the python as build/tests requirement is the testcases.
How hard would be to add a non-python testcase on GLIBC to test such
features?
>
> As a final (not entirely related) point, while I'm talking about
> testing, I'd like to note that while the compiler and tester are both
> works-in-progress, they both have pretty comprehensive test suites.
> The compiler has tests with 98% coverage so I'm pretty sure it's doing
> what I think it is on all supported versions of Python. I'm using
> Travis CI to test every commit I make with seven different versions of
> Python from 2.7 to the latest nightly:
>
> https://travis-ci.org/gbenson/i8c
>
> The tester has only 87% coverage but I'm working on that :)
>
> Cheers,
> Gary
>
Adhemerval Zanella wrote:
> On 21-10-2015 12:42, Gary Benson wrote:
> > Adhemerval Zanella wrote:
> > > I like the idea of improving debugging capabilities with
> > > something new than libpthread_db, however I do not see the best
> > > approach of using the i8 files directly in GLIBC. If I
> > > understood correctly from the link you posted with this
> > > documentation, it is still a very WIP project with rough places
> > > and the language/compiler is still being constantly updated.
> >
> > It is a work-in-progress right now. It's being developed to allow
> > GDB to debug multithreaded inferiors, so this patch series (or,
> > rather, the complete version that will follow in the next few
> > months) is the work that will turn Infinity from WIP to something
> > complete.
> >
> > I mailed this series as an RFA to get some early feedback. I'm
> > anticipating that a prerequisite of a GLIBC commit will be a GDB that
> > fully supports Infinity (and a prerequisite of a GDB commit will be a
> > GLIBC with Infinity notes) and what I *didn't* want to do was appear
> > out of the blue with a complete system I'd spent six months writing
> > away by myself only for somebody to point out a killer flaw that meant
> > the whole thing was junk. Also because I recognise that input from
> > the wider community will make the final result something way better
> > than I could achieve on my own.
> >
> > So yes, it's very work-in-progress right now, but by the time it's
> > ready to commit it should be pretty well baked and polished.
>
> I did say it was flawed or even half-baked, my concern is due it
> uses a non well establish language it may add more maintaining
> burden than use assembly drop releases. What happens if had or
> decided to extend I8X to cover a new feature and add this dependency
> on the GLIBC code? Will you always provide backwards compatibility?
> Will we need to tie GLIBC releases to some I8X release or will be
> feature based?
I wasn't intending to ever break backwards compatibility. That would
mess things up for people (and I'd have to deal with it!)
If some new language feature was added then some configure check could
be written, either by looking at "i8c --version" or by trying to
compile some code that had the new feature and seeing if it failed.
It would be much as we do for the other tools we use.
> > > Also I do not like to add another tool dependency for complete
> > > GLIBC build and one in such state (GLIBC do use python for
> > > benchmark but it is not really required for build afaik). So
> > > which is the issue of shipping the ELF notes in assembly format
> > > instead?
> >
> > The initial reason I opted for a compiled source approach is that
> > the note format is somewhat complex. The format is documented
> > here:
> >
> > https://sourceware.org/gdb/wiki/NoteFormat
> >
> > I want Infinity to be future-proof. Now that libthread_db exists,
> > GLIBC is essentially bound to support it *forever*. I don't want to
> > introduce another debugger interface that we'll have to support
> > forever unless I can be fairly confident we won't have to replace that
> > five or ten years down the line. So there's a lot of requirements in
> > the note format specification that (should!) mean that any new feature
> > we require can be added without breaking existing note consumers.
> >
> > The second reason for using a compiler (or, more generally, tools) is
> > testing. We could ship the notes in assembly language but then how do
> > you test them? I don't like the idea of GLIBC shipping with untested
> > code, especially since the result of a breakage might not be
> > immediately obvious: if we ship a broken note in a GLIBC release it
> > might well not be noticed until it makes its way into some distro and
> > some user finds they can't debug their application because one of the
> > thread-debugging notes is invalid. That's why the required tools are
> > a compiler *and* a unit tester. I don't know if you saw, but the
> > testcase for the note in this patch exercises every path through the
> > compiled note. It's not perfect, but it's as close as I can get. So,
> > each note I add to GLIBC will have an accompanying testcase so I can
> > be as certain as I can be that if "make check" passed then the notes
> > in the files will work.
> >
> > For me, the second reason (testing) is the more compelling of the two.
> > And if you're going to require an external tool for testing, why not
> > require an external compiler too?
>
> AFAIK the only non-trivial build dependency for testing is perl and
> I do not see a way forward to add another interpreter dependency. It
> will be even harder on cross-compiling environment to do native testing
> with such dependency.
I8C and I8X run on the host, not the target. You only need Python on
the host. It's probably already there :)
And on the host, if you have Python and pip installed then you can
install I8C and I8X with a single shell command. It takes about 20
seconds on my <5Mbit connection and you don't need root access.
> > I appreciate that people don't want to add more build dependencies to
> > GLIBC but I don't really have a way around that. To mitigate the
> > impact I've written the tools without using any external libraries,
> > and I've written them to work on as wide a range of Python versions
> > as is possible. It's not possible to write code that works both
> > for Python < 2.6 and for Python >= 3.0 simultaneously so 2.6 is the
> > oldest you can use.
>
> So the compelling reason I see so far to do not make assembly releases
> and to not add the python as build/tests requirement is the testcases.
> How hard would be to add a non-python testcase on GLIBC to test such
> features?
I8X has 1230 statements (lines of code that do something, as opposed
to comments, blank lines and documentation). I don't have figures for
how Python compares to C in terms of line count but I figure you're
looking at 2500-5000 lines of C and that's not including the testcases
themselves or something to parse and execute them if you go that
route. Easily a month of work, and the result would likely be less
flexible.
Cheers,
Gary
On 21-10-2015 17:09, Gary Benson wrote:
> Adhemerval Zanella wrote:
>> On 21-10-2015 12:42, Gary Benson wrote:
>>> Adhemerval Zanella wrote:
>>>> I like the idea of improving debugging capabilities with
>>>> something new than libpthread_db, however I do not see the best
>>>> approach of using the i8 files directly in GLIBC. If I
>>>> understood correctly from the link you posted with this
>>>> documentation, it is still a very WIP project with rough places
>>>> and the language/compiler is still being constantly updated.
>>>
>>> It is a work-in-progress right now. It's being developed to allow
>>> GDB to debug multithreaded inferiors, so this patch series (or,
>>> rather, the complete version that will follow in the next few
>>> months) is the work that will turn Infinity from WIP to something
>>> complete.
>>>
>>> I mailed this series as an RFA to get some early feedback. I'm
>>> anticipating that a prerequisite of a GLIBC commit will be a GDB that
>>> fully supports Infinity (and a prerequisite of a GDB commit will be a
>>> GLIBC with Infinity notes) and what I *didn't* want to do was appear
>>> out of the blue with a complete system I'd spent six months writing
>>> away by myself only for somebody to point out a killer flaw that meant
>>> the whole thing was junk. Also because I recognise that input from
>>> the wider community will make the final result something way better
>>> than I could achieve on my own.
>>>
>>> So yes, it's very work-in-progress right now, but by the time it's
>>> ready to commit it should be pretty well baked and polished.
>>
>> I did say it was flawed or even half-baked, my concern is due it
>> uses a non well establish language it may add more maintaining
>> burden than use assembly drop releases. What happens if had or
>> decided to extend I8X to cover a new feature and add this dependency
>> on the GLIBC code? Will you always provide backwards compatibility?
>> Will we need to tie GLIBC releases to some I8X release or will be
>> feature based?
>
> I wasn't intending to ever break backwards compatibility. That would
> mess things up for people (and I'd have to deal with it!)
>
> If some new language feature was added then some configure check could
> be written, either by looking at "i8c --version" or by trying to
> compile some code that had the new feature and seeing if it failed.
> It would be much as we do for the other tools we use.
>
>>>> Also I do not like to add another tool dependency for complete
>>>> GLIBC build and one in such state (GLIBC do use python for
>>>> benchmark but it is not really required for build afaik). So
>>>> which is the issue of shipping the ELF notes in assembly format
>>>> instead?
>>>
>>> The initial reason I opted for a compiled source approach is that
>>> the note format is somewhat complex. The format is documented
>>> here:
>>>
>>> https://sourceware.org/gdb/wiki/NoteFormat
>>>
>>> I want Infinity to be future-proof. Now that libthread_db exists,
>>> GLIBC is essentially bound to support it *forever*. I don't want to
>>> introduce another debugger interface that we'll have to support
>>> forever unless I can be fairly confident we won't have to replace that
>>> five or ten years down the line. So there's a lot of requirements in
>>> the note format specification that (should!) mean that any new feature
>>> we require can be added without breaking existing note consumers.
>>>
>>> The second reason for using a compiler (or, more generally, tools) is
>>> testing. We could ship the notes in assembly language but then how do
>>> you test them? I don't like the idea of GLIBC shipping with untested
>>> code, especially since the result of a breakage might not be
>>> immediately obvious: if we ship a broken note in a GLIBC release it
>>> might well not be noticed until it makes its way into some distro and
>>> some user finds they can't debug their application because one of the
>>> thread-debugging notes is invalid. That's why the required tools are
>>> a compiler *and* a unit tester. I don't know if you saw, but the
>>> testcase for the note in this patch exercises every path through the
>>> compiled note. It's not perfect, but it's as close as I can get. So,
>>> each note I add to GLIBC will have an accompanying testcase so I can
>>> be as certain as I can be that if "make check" passed then the notes
>>> in the files will work.
>>>
>>> For me, the second reason (testing) is the more compelling of the two.
>>> And if you're going to require an external tool for testing, why not
>>> require an external compiler too?
>>
>> AFAIK the only non-trivial build dependency for testing is perl and
>> I do not see a way forward to add another interpreter dependency. It
>> will be even harder on cross-compiling environment to do native testing
>> with such dependency.
>
> I8C and I8X run on the host, not the target. You only need Python on
> the host. It's probably already there :)
>
> And on the host, if you have Python and pip installed then you can
> install I8C and I8X with a single shell command. It takes about 20
> seconds on my <5Mbit connection and you don't need root access.
That is not the case where you are bootstrapping on a platform that does
not have python enabled.
>
>>> I appreciate that people don't want to add more build dependencies to
>>> GLIBC but I don't really have a way around that. To mitigate the
>>> impact I've written the tools without using any external libraries,
>>> and I've written them to work on as wide a range of Python versions
>>> as is possible. It's not possible to write code that works both
>>> for Python < 2.6 and for Python >= 3.0 simultaneously so 2.6 is the
>>> oldest you can use.
>>
>> So the compelling reason I see so far to do not make assembly releases
>> and to not add the python as build/tests requirement is the testcases.
>> How hard would be to add a non-python testcase on GLIBC to test such
>> features?
>
> I8X has 1230 statements (lines of code that do something, as opposed
> to comments, blank lines and documentation). I don't have figures for
> how Python compares to C in terms of line count but I figure you're
> looking at 2500-5000 lines of C and that's not including the testcases
> themselves or something to parse and execute them if you go that
> route. Easily a month of work, and the result would likely be less
> flexible.
I was thinking more like something a objdump or similar to just dump
the notes in a meaningful way.
Anyway, I my only concern is setting python as a glibc dependency build
where IHMO does not really add anything. I think one strategy that we
can use to decouple the need for a python binary in build is:
1. Provide assembly generated notes from a defined I8X compiler along
with I8X sources.
2. Add a makefile rule to autogenerate the assembly file if a I8C
compiler is presented in the system.
3. For tests, only run is I8X is presented.
>
> Cheers,
> Gary
>
Adhemerval Zanella wrote:
> On 21-10-2015 17:09, Gary Benson wrote:
> > Adhemerval Zanella wrote:
> > > AFAIK the only non-trivial build dependency for testing is perl
> > > and I do not see a way forward to add another interpreter
> > > dependency. It will be even harder on cross-compiling
> > > environment to do native testing with such dependency.
> >
> > I8C and I8X run on the host, not the target. You only need Python
> > on the host. It's probably already there :)
> >
> > And on the host, if you have Python and pip installed then you can
> > install I8C and I8X with a single shell command. It takes about
> > 20 seconds on my <5Mbit connection and you don't need root access.
>
> That is not the case where you are bootstrapping on a platform that
> does not have python enabled.
Don't people usually cross-build to bootstrap new platforms?
> > > So the compelling reason I see so far to do not make assembly
> > > releases and to not add the python as build/tests requirement is
> > > the testcases. How hard would be to add a non-python testcase
> > > on GLIBC to test such features?
> >
> > I8X has 1230 statements (lines of code that do something, as
> > opposed to comments, blank lines and documentation). I don't have
> > figures for how Python compares to C in terms of line count but I
> > figure you're looking at 2500-5000 lines of C and that's not
> > including the testcases themselves or something to parse and
> > execute them if you go that route. Easily a month of work, and
> > the result would likely be less flexible.
>
> I was thinking more like something a objdump or similar to just dump
> the notes in a meaningful way.
With respect that's not really the same thing at all. The I8X
testcase executes the functions multiple times with the correct
endianness and wordsize for the target. The generated functions
are different for each platform too--they have structure offsets
and other constants embedded in them, for example, and the note
in this patch has three different variants depending on how the
target does THREAD_SELF. You can't even compare the operations
because if a field in a structure has a zero offset on one
platform and a nonzero offset on another then the platform with
the nonzero offset will have extra adds that the compiler
optimized away on the platform with the zero offset.
> Anyway, I my only concern is setting python as a glibc dependency
> build where IHMO does not really add anything.
I have to disagree with you there. My opinion is that it adds a
great deal. See the paragraph above and the paragraph below.
> I think one strategy that we can use to decouple the need for a
> python binary in build is:
>
> 1. Provide assembly generated notes from a defined I8X compiler
> along with I8X sources.
> 2. Add a makefile rule to autogenerate the assembly file if a I8C
> compiler is presented in the system.
> 3. For tests, only run is I8X is presented.
That's not really viable for the reasons I detailed above. You would
have to ship generated assembly for every platform, and even that is
not enough. For example, on some platforms "sizeof (struct pthread)"
is a constant that gets into notes. If someone adds a field to struct
pthread and doesn't regenerate the notes then you're shipping GLIBC
with broken notes. Are there CFLAGS the user could set that would
change structure offsets? Setting these would mean broken notes if
you didn't have I8C installed. There are constants in kernel headers
that end up in notes. If one of these changes, you're shipping GLIBC
with broken notes.
There are so many silent failure modes here--and broken notes are far
worse than no notes at all. The only robust way to do this is to
build the notes with the constants and offsets that you are building
the rest of GLIBC with, and that means saying if you want a GLIBC
with notes then you need to install this package on your build
system.
Cheers,
Gary
> Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
>
> Adhemerval Zanella wrote:
>>> On 21-10-2015 17:09, Gary Benson wrote:
>>> Adhemerval Zanella wrote:
>>>> AFAIK the only non-trivial build dependency for testing is perl
>>>> and I do not see a way forward to add another interpreter
>>>> dependency. It will be even harder on cross-compiling
>>>> environment to do native testing with such dependency.
>>>
>>> I8C and I8X run on the host, not the target. You only need Python
>>> on the host. It's probably already there :)
>>>
>>> And on the host, if you have Python and pip installed then you can
>>> install I8C and I8X with a single shell command. It takes about
>>> 20 seconds on my <5Mbit connection and you don't need root access.
>>
>> That is not the case where you are bootstrapping on a platform that
>> does not have python enabled.
>
> Don't people usually cross-build to bootstrap new platforms?
>
So the testing itself does not require to run Python natively? If it is the case where you can the i8x tests in the build machine I agree that it is not an impending reason.
>>>> So the compelling reason I see so far to do not make assembly
>>>> releases and to not add the python as build/tests requirement is
>>>> the testcases. How hard would be to add a non-python testcase
>>>> on GLIBC to test such features?
>>>
>>> I8X has 1230 statements (lines of code that do something, as
>>> opposed to comments, blank lines and documentation). I don't have
>>> figures for how Python compares to C in terms of line count but I
>>> figure you're looking at 2500-5000 lines of C and that's not
>>> including the testcases themselves or something to parse and
>>> execute them if you go that route. Easily a month of work, and
>>> the result would likely be less flexible.
>>
>> I was thinking more like something a objdump or similar to just dump
>> the notes in a meaningful way.
>
> With respect that's not really the same thing at all. The I8X
> testcase executes the functions multiple times with the correct
> endianness and wordsize for the target. The generated functions
> are different for each platform too--they have structure offsets
> and other constants embedded in them, for example, and the note
> in this patch has three different variants depending on how the
> target does THREAD_SELF. You can't even compare the operations
> because if a field in a structure has a zero offset on one
> platform and a nonzero offset on another then the platform with
> the nonzero offset will have extra adds that the compiler
> optimized away on the platform with the zero offset.
Unless it requires runtime information where you can't obtain prior hand due kernel/hardware interaction I do not see why this is not a matter of parse the correct fields and information in the notes.
However I see that the infrastructure is already created using i8x, so it would be a duplicate effort.
>
>> Anyway, I my only concern is setting python as a glibc dependency
>> build where IHMO does not really add anything.
>
> I have to disagree with you there. My opinion is that it adds a
> great deal. See the paragraph above and the paragraph below.
>
>> I think one strategy that we can use to decouple the need for a
>> python binary in build is:
>>
>> 1. Provide assembly generated notes from a defined I8X compiler
>> along with I8X sources.
>> 2. Add a makefile rule to autogenerate the assembly file if a I8C
>> compiler is presented in the system.
>> 3. For tests, only run is I8X is presented.
>
> That's not really viable for the reasons I detailed above. You would
> have to ship generated assembly for every platform, and even that is
> not enough. For example, on some platforms "sizeof (struct pthread)"
> is a constant that gets into notes. If someone adds a field to struct
> pthread and doesn't regenerate the notes then you're shipping GLIBC
> with broken notes. Are there CFLAGS the user could set that would
> change structure offsets? Setting these would mean broken notes if
> you didn't have I8C installed. There are constants in kernel headers
> that end up in notes. If one of these changes, you're shipping GLIBC
> with broken notes.
>
> There are so many silent failure modes here--and broken notes are far
> worse than no notes at all. The only robust way to do this is to
> build the notes with the constants and offsets that you are building
> the rest of GLIBC with, and that means saying if you want a GLIBC
> with notes then you need to install this package on your build
> system.
>
This can be done with proper documentation and in the release cycle: the release maintainer will recreate the assembly annotations for the release architectures.
My only concern is the kernel notes dependency, however even those we need to take care since you also do not know prior hand which version you will end using.
Also we can use the same strategy with the ulp updates for link testing.
> Cheers,
> Gary
>
> --
> http://gbenson.net/
Adhemerval Zanella wrote:
> > Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
> > Adhemerval Zanella wrote:
> > > > On 21-10-2015 17:09, Gary Benson wrote:
> > > > Adhemerval Zanella wrote:
> > > > > AFAIK the only non-trivial build dependency for testing
> > > > > is perl and I do not see a way forward to add another
> > > > > interpreter dependency. It will be even harder on
> > > > > cross-compiling environment to do native testing with such
> > > > > dependency.
> > > >
> > > > I8C and I8X run on the host, not the target. You only need
> > > > Python on the host. It's probably already there :)
> > > >
> > > > And on the host, if you have Python and pip installed then you
> > > > can install I8C and I8X with a single shell command. It takes
> > > > about 20 seconds on my <5Mbit connection and you don't need
> > > > root access.
> > >
> > > That is not the case where you are bootstrapping on a platform
> > > that does not have python enabled.
> >
> > Don't people usually cross-build to bootstrap new platforms?
>
> So the testing itself does not require to run Python natively? If it
> is the case where you can the i8x tests in the build machine I agree
> that it is not an impending reason.
Neither the compile step nor the test step require Python to run
natively. I8C and I8X both run only on the build host.
> > > > > So the compelling reason I see so far to do not make
> > > > > assembly releases and to not add the python as build/tests
> > > > > requirement is the testcases. How hard would be to add a
> > > > > non-python testcase on GLIBC to test such features?
> > > >
> > > > I8X has 1230 statements (lines of code that do something, as
> > > > opposed to comments, blank lines and documentation). I don't
> > > > have figures for how Python compares to C in terms of line
> > > > count but I figure you're looking at 2500-5000 lines of C and
> > > > that's not including the testcases themselves or something to
> > > > parse and execute them if you go that route. Easily a month
> > > > of work, and the result would likely be less flexible.
> > >
> > > I was thinking more like something a objdump or similar to just
> > > dump the notes in a meaningful way.
> >
> > With respect that's not really the same thing at all. The I8X
> > testcase executes the functions multiple times with the correct
> > endianness and wordsize for the target. The generated functions
> > are different for each platform too--they have structure offsets
> > and other constants embedded in them, for example, and the note
> > in this patch has three different variants depending on how the
> > target does THREAD_SELF. You can't even compare the operations
> > because if a field in a structure has a zero offset on one
> > platform and a nonzero offset on another then the platform with
> > the nonzero offset will have extra adds that the compiler
> > optimized away on the platform with the zero offset.
>
> Unless it requires runtime information where you can't obtain prior
> hand due kernel/hardware interaction I do not see why this is not a
> matter of parse the correct fields and information in the notes.
Sure, but what you're talking about is essentially recreating the
note compiler. I8C is currently 2699 Python statements plus a 917
statement testsuite, so over 3500 Python statements to translate
into a presumably less-expressive language.
> However I see that the infrastructure is already created using i8x,
> so it would be a duplicate effort.
Yeah. And I8X is testing a different thing too. It's not checking
the bytecode you got is the bytecode you expected, it's checking the
result of executing the bytecode is the result you expected.
> > > I think one strategy that we can use to decouple the need for a
> > > python binary in build is:
> > >
> > > 1. Provide assembly generated notes from a defined I8X compiler
> > > along with I8X sources.
> > > 2. Add a makefile rule to autogenerate the assembly file if a I8C
> > > compiler is presented in the system.
> > > 3. For tests, only run is I8X is presented.
> >
> > That's not really viable for the reasons I detailed above. You
> > would have to ship generated assembly for every platform, and even
> > that is not enough. For example, on some platforms "sizeof
> > (struct pthread)" is a constant that gets into notes. If someone
> > adds a field to struct pthread and doesn't regenerate the notes
> > then you're shipping GLIBC with broken notes. Are there CFLAGS
> > the user could set that would change structure offsets? Setting
> > these would mean broken notes if you didn't have I8C installed.
> > There are constants in kernel headers that end up in notes. If
> > one of these changes, you're shipping GLIBC with broken notes.
> >
> > There are so many silent failure modes here--and broken notes are
> > far worse than no notes at all. The only robust way to do this is
> > to build the notes with the constants and offsets that you are
> > building the rest of GLIBC with, and that means saying if you want
> > a GLIBC with notes then you need to install this package on your
> > build system.
>
> This can be done with proper documentation and in the release cycle:
> the release maintainer will recreate the assembly annotations for
> the release architectures. My only concern is the kernel notes
> dependency, however even those we need to take care since you also
> do not know prior hand which version you will end using.
That's fragile. It's also burdensome for the release maintainer.
It also raises the barrier for people hacking on GLIBC, because not
only do they need to know all the other things they need to know, they
also now need to know about recreating notes. Say somebody is hacking
on NPTL, they don't know about updating notes, so the first thing they
see is the debugger won't attach. How long do they spend going round
in circles until somebody says "oh, you need to run this command in
this directory."?
It also doesn't solve the problem that if people are working without
I8C and I8X installed then they are creating GLIBCs containing
unchecked notes. My position is users create GLIBCs containing
fully-tested notes or they create GLIBCs creating no notes at all.
In either setup there's a process to create notes. What you're
saying is, "let's encode this process in documentation and let
people figure it out." What I'm saying is, "let's encode this in
code and let the build host figure it out."
Thanks,
Gary
On 22/10/15 09:56, Gary Benson wrote:
> Adhemerval Zanella wrote:
>>> Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
>>> Adhemerval Zanella wrote:
>>>>> On 21-10-2015 17:09, Gary Benson wrote:
>>>>> Adhemerval Zanella wrote:
>>>>>> AFAIK the only non-trivial build dependency for testing
>>>>>> is perl and I do not see a way forward to add another
>>>>>> interpreter dependency. It will be even harder on
>>>>>> cross-compiling environment to do native testing with such
>>>>>> dependency.
>>>>>
>>>>> I8C and I8X run on the host, not the target. You only need
>>>>> Python on the host. It's probably already there :)
>>>>>
>>>>> And on the host, if you have Python and pip installed then you
>>>>> can install I8C and I8X with a single shell command. It takes
>>>>> about 20 seconds on my <5Mbit connection and you don't need
>>>>> root access.
>>>>
>>>> That is not the case where you are bootstrapping on a platform
>>>> that does not have python enabled.
>>>
>>> Don't people usually cross-build to bootstrap new platforms?
>>
>> So the testing itself does not require to run Python natively? If it
>> is the case where you can the i8x tests in the build machine I agree
>> that it is not an impending reason.
>
> Neither the compile step nor the test step require Python to run
> natively. I8C and I8X both run only on the build host.
>
the easiest way to run the glibc tests is to build it natively
(because the test system has assumptions about the host env that
is hard to match with the ssh+nfs cross test setup).
currently the test system depends on perl (for header conformance
tests and memory leak tests) which is already a pain.
python is another hard to build dependency and you have to make
sure that python language changes don't affect the behaviour.
i think this is not a critical issue since the build can succeed
without i8*, but it is worth noting.
On 22-10-2015 06:56, Gary Benson wrote:
> Adhemerval Zanella wrote:
>>> Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
>>> Adhemerval Zanella wrote:
>>>>> On 21-10-2015 17:09, Gary Benson wrote:
>>>>> Adhemerval Zanella wrote:
>>>>>> AFAIK the only non-trivial build dependency for testing
>>>>>> is perl and I do not see a way forward to add another
>>>>>> interpreter dependency. It will be even harder on
>>>>>> cross-compiling environment to do native testing with such
>>>>>> dependency.
>>>>>
>>>>> I8C and I8X run on the host, not the target. You only need
>>>>> Python on the host. It's probably already there :)
>>>>>
>>>>> And on the host, if you have Python and pip installed then you
>>>>> can install I8C and I8X with a single shell command. It takes
>>>>> about 20 seconds on my <5Mbit connection and you don't need
>>>>> root access.
>>>>
>>>> That is not the case where you are bootstrapping on a platform
>>>> that does not have python enabled.
>>>
>>> Don't people usually cross-build to bootstrap new platforms?
>>
>> So the testing itself does not require to run Python natively? If it
>> is the case where you can the i8x tests in the build machine I agree
>> that it is not an impending reason.
>
> Neither the compile step nor the test step require Python to run
> natively. I8C and I8X both run only on the build host.
>
>>>>>> So the compelling reason I see so far to do not make
>>>>>> assembly releases and to not add the python as build/tests
>>>>>> requirement is the testcases. How hard would be to add a
>>>>>> non-python testcase on GLIBC to test such features?
>>>>>
>>>>> I8X has 1230 statements (lines of code that do something, as
>>>>> opposed to comments, blank lines and documentation). I don't
>>>>> have figures for how Python compares to C in terms of line
>>>>> count but I figure you're looking at 2500-5000 lines of C and
>>>>> that's not including the testcases themselves or something to
>>>>> parse and execute them if you go that route. Easily a month
>>>>> of work, and the result would likely be less flexible.
>>>>
>>>> I was thinking more like something a objdump or similar to just
>>>> dump the notes in a meaningful way.
>>>
>>> With respect that's not really the same thing at all. The I8X
>>> testcase executes the functions multiple times with the correct
>>> endianness and wordsize for the target. The generated functions
>>> are different for each platform too--they have structure offsets
>>> and other constants embedded in them, for example, and the note
>>> in this patch has three different variants depending on how the
>>> target does THREAD_SELF. You can't even compare the operations
>>> because if a field in a structure has a zero offset on one
>>> platform and a nonzero offset on another then the platform with
>>> the nonzero offset will have extra adds that the compiler
>>> optimized away on the platform with the zero offset.
>>
>> Unless it requires runtime information where you can't obtain prior
>> hand due kernel/hardware interaction I do not see why this is not a
>> matter of parse the correct fields and information in the notes.
>
> Sure, but what you're talking about is essentially recreating the
> note compiler. I8C is currently 2699 Python statements plus a 917
> statement testsuite, so over 3500 Python statements to translate
> into a presumably less-expressive language.
>
>> However I see that the infrastructure is already created using i8x,
>> so it would be a duplicate effort.
>
> Yeah. And I8X is testing a different thing too. It's not checking
> the bytecode you got is the bytecode you expected, it's checking the
> result of executing the bytecode is the result you expected.
That's why I asked if you need to run natively, since the expected results
will only vary if you have runtime constraints. But I do see that changing
the offsets or internal structures will probably ending to adjust the
tests expectations also.
>
>>>> I think one strategy that we can use to decouple the need for a
>>>> python binary in build is:
>>>>
>>>> 1. Provide assembly generated notes from a defined I8X compiler
>>>> along with I8X sources.
>>>> 2. Add a makefile rule to autogenerate the assembly file if a I8C
>>>> compiler is presented in the system.
>>>> 3. For tests, only run is I8X is presented.
>>>
>>> That's not really viable for the reasons I detailed above. You
>>> would have to ship generated assembly for every platform, and even
>>> that is not enough. For example, on some platforms "sizeof
>>> (struct pthread)" is a constant that gets into notes. If someone
>>> adds a field to struct pthread and doesn't regenerate the notes
>>> then you're shipping GLIBC with broken notes. Are there CFLAGS
>>> the user could set that would change structure offsets? Setting
>>> these would mean broken notes if you didn't have I8C installed.
>>> There are constants in kernel headers that end up in notes. If
>>> one of these changes, you're shipping GLIBC with broken notes.
>>>
>>> There are so many silent failure modes here--and broken notes are
>>> far worse than no notes at all. The only robust way to do this is
>>> to build the notes with the constants and offsets that you are
>>> building the rest of GLIBC with, and that means saying if you want
>>> a GLIBC with notes then you need to install this package on your
>>> build system.
>>
>> This can be done with proper documentation and in the release cycle:
>> the release maintainer will recreate the assembly annotations for
>> the release architectures. My only concern is the kernel notes
>> dependency, however even those we need to take care since you also
>> do not know prior hand which version you will end using.
>
> That's fragile. It's also burdensome for the release maintainer.
> It also raises the barrier for people hacking on GLIBC, because not
> only do they need to know all the other things they need to know, they
> also now need to know about recreating notes. Say somebody is hacking
> on NPTL, they don't know about updating notes, so the first thing they
> see is the debugger won't attach. How long do they spend going round
> in circles until somebody says "oh, you need to run this command in
> this directory."?
>
> It also doesn't solve the problem that if people are working without
> I8C and I8X installed then they are creating GLIBCs containing
> unchecked notes. My position is users create GLIBCs containing
> fully-tested notes or they create GLIBCs creating no notes at all.
>
> In either setup there's a process to create notes. What you're
> saying is, "let's encode this process in documentation and let
> people figure it out." What I'm saying is, "let's encode this in
> code and let the build host figure it out."
Yes it is a tradeoff of adding external dependency and making GLIBC
build independently of them. Also the exported NPTL structures do
not change in a dynamic way (struct pthread, for instance, changes
it's size twice in seven years).
Also, currently if someone is hacking on GLIBC and he will also need
to adjust libpthread_db anyway. I know it is not the best approach
and I do want to see a better strategy to handle it, but adding an
external tools with python dependency IMHO is not the best one.
I would expect such tool to be included in binutils and have only
binutils own dependencies as requisites. I would also expect notes
definition language stability, meaning that a notes defined with a
older version will always be handled by new compiler version
(so building a older glibc notes version with a new notes compiler
won't break the build).
>
> Thanks,
> Gary
>
Adhemerval Zanella wrote:
> On 22-10-2015 06:56, Gary Benson wrote:
> > Adhemerval Zanella wrote:
> > > Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
> > > > Adhemerval Zanella wrote:
> > > > > I think one strategy that we can use to decouple the need
> > > > > for a python binary in build is:
> > > > >
> > > > > 1. Provide assembly generated notes from a defined I8X
> > > > > compiler along with I8X sources.
> > > > > 2. Add a makefile rule to autogenerate the assembly file
> > > > > if a I8C compiler is presented in the system.
> > > > > 3. For tests, only run is I8X is presented.
> > > >
> > > > That's not really viable for the reasons I detailed above.
> > > > You would have to ship generated assembly for every platform,
> > > > and even that is not enough. For example, on some platforms
> > > > "sizeof (struct pthread)" is a constant that gets into notes.
> > > > If someone adds a field to struct pthread and doesn't
> > > > regenerate the notes then you're shipping GLIBC with broken
> > > > notes. Are there CFLAGS the user could set that would change
> > > > structure offsets? Setting these would mean broken notes if
> > > > you didn't have I8C installed. There are constants in kernel
> > > > headers that end up in notes. If one of these changes, you're
> > > > shipping GLIBC with broken notes.
> > > >
> > > > There are so many silent failure modes here--and broken notes
> > > > are far worse than no notes at all. The only robust way to do
> > > > this is to build the notes with the constants and offsets that
> > > > you are building the rest of GLIBC with, and that means saying
> > > > if you want a GLIBC with notes then you need to install this
> > > > package on your build system.
> > >
> > > This can be done with proper documentation and in the release
> > > cycle: the release maintainer will recreate the assembly
> > > annotations for the release architectures. My only concern is
> > > the kernel notes dependency, however even those we need to take
> > > care since you also do not know prior hand which version you
> > > will end using.
> >
> > That's fragile. It's also burdensome for the release maintainer.
> > It also raises the barrier for people hacking on GLIBC, because
> > not only do they need to know all the other things they need to
> > know, they also now need to know about recreating notes. Say
> > somebody is hacking on NPTL, they don't know about updating notes,
> > so the first thing they see is the debugger won't attach. How
> > long do they spend going round in circles until somebody says "oh,
> > you need to run this command in this directory."?
> >
> > It also doesn't solve the problem that if people are working
> > without I8C and I8X installed then they are creating GLIBCs
> > containing unchecked notes. My position is users create GLIBCs
> > containing fully-tested notes or they create GLIBCs creating no
> > notes at all.
> >
> > In either setup there's a process to create notes. What you're
> > saying is, "let's encode this process in documentation and let
> > people figure it out." What I'm saying is, "let's encode this in
> > code and let the build host figure it out."
>
> Yes it is a tradeoff of adding external dependency and making GLIBC
> build independently of them. Also the exported NPTL structures do
> not change in a dynamic way (struct pthread, for instance, changes
> it's size twice in seven years).
I've given a bunch of reasons why pre-generated notes might become
broken. The fact that *one* of those reasons might not occur very
often is irrelevant.
The impact of a broken note in an application is that debugging that
application may be difficult or impossible. GLIBC is used by almost
every application, so a broken note in GLIBC could mean it becomes
impossible to debug *almost any application* on your system.
Including the debugger which you might like to use to locate the
problem. So it's important that any notes in GLIBC are thoroughly
tested.
Pregenerated notes have identified failure modes which cannot be
detected at generation time. The only way to detect such failures
is by testing the build.
Notes are complex enough that they cannot be thoroughly tested
without tooling. If that tooling is not present, "make check"
cannot check the build.
The impact of broken notes is sufficiently severe that if notes
cannot be tested then they should not be present in the build.
So you require tooling to have a build with notes. And if you
require tooling to have a build with notes, you may as well
have tooling to build the notes in the first place.
> Also, currently if someone is hacking on GLIBC and he will also need
> to adjust libpthread_db anyway. I know it is not the best approach
> and I do want to see a better strategy to handle it, but adding an
> external tools with python dependency IMHO is not the best one.
I hope this isn't all about Python. Python is pretty mainstream,
and I'm pretty sure most people will either have it on their build
system already or could install it with a single shell command.
I appreciate that people who are cross-compiling may not wish to
or be able to install Python on their target machines, which is
why I've taken steps to ensure that Python is only used on the
host.
> I would expect such tool to be included in binutils and have only
> binutils own dependencies as requisites.
GLIBC has plenty of dependencies from outside of binutils.
I'm happy for the project to be donated to the FSF and/or hosted
on sourceware.org if that would reassure people. I don't think,
however, that binutils is a natural place to put a Python project.
> I would also expect notes definition language stability, meaning
> that a notes defined with a older version will always be handled
> by new compiler version (so building a older glibc notes version
> with a new notes compiler won't break the build).
Naturally. And there's no reason for that not to be the case.
Thanks,
Gary
On 22-10-2015 11:11, Gary Benson wrote:
> Adhemerval Zanella wrote:
>> On 22-10-2015 06:56, Gary Benson wrote:
>>> Adhemerval Zanella wrote:
>>>> Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
>>>>> Adhemerval Zanella wrote:
>>>>>> I think one strategy that we can use to decouple the need
>>>>>> for a python binary in build is:
>>>>>>
>>>>>> 1. Provide assembly generated notes from a defined I8X
>>>>>> compiler along with I8X sources.
>>>>>> 2. Add a makefile rule to autogenerate the assembly file
>>>>>> if a I8C compiler is presented in the system.
>>>>>> 3. For tests, only run is I8X is presented.
>>>>>
>>>>> That's not really viable for the reasons I detailed above.
>>>>> You would have to ship generated assembly for every platform,
>>>>> and even that is not enough. For example, on some platforms
>>>>> "sizeof (struct pthread)" is a constant that gets into notes.
>>>>> If someone adds a field to struct pthread and doesn't
>>>>> regenerate the notes then you're shipping GLIBC with broken
>>>>> notes. Are there CFLAGS the user could set that would change
>>>>> structure offsets? Setting these would mean broken notes if
>>>>> you didn't have I8C installed. There are constants in kernel
>>>>> headers that end up in notes. If one of these changes, you're
>>>>> shipping GLIBC with broken notes.
>>>>>
>>>>> There are so many silent failure modes here--and broken notes
>>>>> are far worse than no notes at all. The only robust way to do
>>>>> this is to build the notes with the constants and offsets that
>>>>> you are building the rest of GLIBC with, and that means saying
>>>>> if you want a GLIBC with notes then you need to install this
>>>>> package on your build system.
>>>>
>>>> This can be done with proper documentation and in the release
>>>> cycle: the release maintainer will recreate the assembly
>>>> annotations for the release architectures. My only concern is
>>>> the kernel notes dependency, however even those we need to take
>>>> care since you also do not know prior hand which version you
>>>> will end using.
>>>
>>> That's fragile. It's also burdensome for the release maintainer.
>>> It also raises the barrier for people hacking on GLIBC, because
>>> not only do they need to know all the other things they need to
>>> know, they also now need to know about recreating notes. Say
>>> somebody is hacking on NPTL, they don't know about updating notes,
>>> so the first thing they see is the debugger won't attach. How
>>> long do they spend going round in circles until somebody says "oh,
>>> you need to run this command in this directory."?
>>>
>>> It also doesn't solve the problem that if people are working
>>> without I8C and I8X installed then they are creating GLIBCs
>>> containing unchecked notes. My position is users create GLIBCs
>>> containing fully-tested notes or they create GLIBCs creating no
>>> notes at all.
>>>
>>> In either setup there's a process to create notes. What you're
>>> saying is, "let's encode this process in documentation and let
>>> people figure it out." What I'm saying is, "let's encode this in
>>> code and let the build host figure it out."
>>
>> Yes it is a tradeoff of adding external dependency and making GLIBC
>> build independently of them. Also the exported NPTL structures do
>> not change in a dynamic way (struct pthread, for instance, changes
>> it's size twice in seven years).
>
> I've given a bunch of reasons why pre-generated notes might become
> broken. The fact that *one* of those reasons might not occur very
> often is irrelevant.
>
> The impact of a broken note in an application is that debugging that
> application may be difficult or impossible. GLIBC is used by almost
> every application, so a broken note in GLIBC could mean it becomes
> impossible to debug *almost any application* on your system.
> Including the debugger which you might like to use to locate the
> problem. So it's important that any notes in GLIBC are thoroughly
> tested.
>
> Pregenerated notes have identified failure modes which cannot be
> detected at generation time. The only way to detect such failures
> is by testing the build.
>
> Notes are complex enough that they cannot be thoroughly tested
> without tooling. If that tooling is not present, "make check"
> cannot check the build.
>
> The impact of broken notes is sufficiently severe that if notes
> cannot be tested then they should not be present in the build.
> So you require tooling to have a build with notes. And if you
> require tooling to have a build with notes, you may as well
> have tooling to build the notes in the first place.
Yes I understand the issue, so in the end it is a matter to allow
external optional tools to add marks and hook up in GLIBC. I would
prefer something related to systemtap way, however it also have its
defects (testing for instance).
I would prefer something more decoupled of GLIBC, so if we need to
get rid off its support if a new strategy/project shows a superior
approach it would be easier. I would also prefer to no be dependent
of external interpreters to implement such compiler/instrumentation
(since IMHO there is no strong reason to depend of python or any
other interpreter for such project).
But also this I also see it is not a strong reason to really oppose
it.
>
>> Also, currently if someone is hacking on GLIBC and he will also need
>> to adjust libpthread_db anyway. I know it is not the best approach
>> and I do want to see a better strategy to handle it, but adding an
>> external tools with python dependency IMHO is not the best one.
>
> I hope this isn't all about Python. Python is pretty mainstream,
> and I'm pretty sure most people will either have it on their build
> system already or could install it with a single shell command.
> I appreciate that people who are cross-compiling may not wish to
> or be able to install Python on their target machines, which is
> why I've taken steps to ensure that Python is only used on the
> host.
>
>> I would expect such tool to be included in binutils and have only
>> binutils own dependencies as requisites.
>
> GLIBC has plenty of dependencies from outside of binutils.
>
> I'm happy for the project to be donated to the FSF and/or hosted
> on sourceware.org if that would reassure people. I don't think,
> however, that binutils is a natural place to put a Python project.
>
>> I would also expect notes definition language stability, meaning
>> that a notes defined with a older version will always be handled
>> by new compiler version (so building a older glibc notes version
>> with a new notes compiler won't break the build).
>
> Naturally. And there's no reason for that not to be the case.
>
> Thanks,
> Gary
>
On 22-10-2015 11:11, Gary Benson wrote:
> Adhemerval Zanella wrote:
>> On 22-10-2015 06:56, Gary Benson wrote:
>>> Adhemerval Zanella wrote:
>>>> Em 21 de out de 2015, às 19:28, Gary Benson <gbenson@redhat.com> escreveu:
>>>>> Adhemerval Zanella wrote:
>>>>>> I think one strategy that we can use to decouple the need
>>>>>> for a python binary in build is:
>>>>>>
>>>>>> 1. Provide assembly generated notes from a defined I8X
>>>>>> compiler along with I8X sources.
>>>>>> 2. Add a makefile rule to autogenerate the assembly file
>>>>>> if a I8C compiler is presented in the system.
>>>>>> 3. For tests, only run is I8X is presented.
>>>>>
>>>>> That's not really viable for the reasons I detailed above.
>>>>> You would have to ship generated assembly for every platform,
>>>>> and even that is not enough. For example, on some platforms
>>>>> "sizeof (struct pthread)" is a constant that gets into notes.
>>>>> If someone adds a field to struct pthread and doesn't
>>>>> regenerate the notes then you're shipping GLIBC with broken
>>>>> notes. Are there CFLAGS the user could set that would change
>>>>> structure offsets? Setting these would mean broken notes if
>>>>> you didn't have I8C installed. There are constants in kernel
>>>>> headers that end up in notes. If one of these changes, you're
>>>>> shipping GLIBC with broken notes.
>>>>>
>>>>> There are so many silent failure modes here--and broken notes
>>>>> are far worse than no notes at all. The only robust way to do
>>>>> this is to build the notes with the constants and offsets that
>>>>> you are building the rest of GLIBC with, and that means saying
>>>>> if you want a GLIBC with notes then you need to install this
>>>>> package on your build system.
>>>>
>>>> This can be done with proper documentation and in the release
>>>> cycle: the release maintainer will recreate the assembly
>>>> annotations for the release architectures. My only concern is
>>>> the kernel notes dependency, however even those we need to take
>>>> care since you also do not know prior hand which version you
>>>> will end using.
>>>
>>> That's fragile. It's also burdensome for the release maintainer.
>>> It also raises the barrier for people hacking on GLIBC, because
>>> not only do they need to know all the other things they need to
>>> know, they also now need to know about recreating notes. Say
>>> somebody is hacking on NPTL, they don't know about updating notes,
>>> so the first thing they see is the debugger won't attach. How
>>> long do they spend going round in circles until somebody says "oh,
>>> you need to run this command in this directory."?
>>>
>>> It also doesn't solve the problem that if people are working
>>> without I8C and I8X installed then they are creating GLIBCs
>>> containing unchecked notes. My position is users create GLIBCs
>>> containing fully-tested notes or they create GLIBCs creating no
>>> notes at all.
>>>
>>> In either setup there's a process to create notes. What you're
>>> saying is, "let's encode this process in documentation and let
>>> people figure it out." What I'm saying is, "let's encode this in
>>> code and let the build host figure it out."
>>
>> Yes it is a tradeoff of adding external dependency and making GLIBC
>> build independently of them. Also the exported NPTL structures do
>> not change in a dynamic way (struct pthread, for instance, changes
>> it's size twice in seven years).
>
> I've given a bunch of reasons why pre-generated notes might become
> broken. The fact that *one* of those reasons might not occur very
> often is irrelevant.
>
> The impact of a broken note in an application is that debugging that
> application may be difficult or impossible. GLIBC is used by almost
> every application, so a broken note in GLIBC could mean it becomes
> impossible to debug *almost any application* on your system.
> Including the debugger which you might like to use to locate the
> problem. So it's important that any notes in GLIBC are thoroughly
> tested.
>
> Pregenerated notes have identified failure modes which cannot be
> detected at generation time. The only way to detect such failures
> is by testing the build.
>
> Notes are complex enough that they cannot be thoroughly tested
> without tooling. If that tooling is not present, "make check"
> cannot check the build.
>
> The impact of broken notes is sufficiently severe that if notes
> cannot be tested then they should not be present in the build.
> So you require tooling to have a build with notes. And if you
> require tooling to have a build with notes, you may as well
> have tooling to build the notes in the first place.
Yes I understand the issue, so in the end it is a matter to allow
external optional tools to add marks and hook up in GLIBC. I would
prefer something related to systemtap way, however it also have its
defects (testing for instance).
I would prefer something more decoupled of GLIBC, so if we need to
get rid off its support if a new strategy/project shows a superior
approach it would be easier. I would also prefer to no be dependent
of external interpreters to implement such compiler/instrumentation
(since IMHO there is no strong reason to depend of python or any
other interpreter for such project).
But also this I also see it is not a strong reason to really oppose
it.
>
>> Also, currently if someone is hacking on GLIBC and he will also need
>> to adjust libpthread_db anyway. I know it is not the best approach
>> and I do want to see a better strategy to handle it, but adding an
>> external tools with python dependency IMHO is not the best one.
>
> I hope this isn't all about Python. Python is pretty mainstream,
> and I'm pretty sure most people will either have it on their build
> system already or could install it with a single shell command.
> I appreciate that people who are cross-compiling may not wish to
> or be able to install Python on their target machines, which is
> why I've taken steps to ensure that Python is only used on the
> host.
>
>> I would expect such tool to be included in binutils and have only
>> binutils own dependencies as requisites.
>
> GLIBC has plenty of dependencies from outside of binutils.
>
> I'm happy for the project to be donated to the FSF and/or hosted
> on sourceware.org if that would reassure people. I don't think,
> however, that binutils is a natural place to put a Python project.
>
>> I would also expect notes definition language stability, meaning
>> that a notes defined with a older version will always be handled
>> by new compiler version (so building a older glibc notes version
>> with a new notes compiler won't break the build).
>
> Naturally. And there's no reason for that not to be the case.
>
> Thanks,
> Gary
>
Szabolcs Nagy wrote:
> On 22/10/15 09:56, Gary Benson wrote:
> > Don't people usually cross-build to bootstrap new platforms?
>
> the easiest way to run the glibc tests is to build it natively
> (because the test system has assumptions about the host env that
> is hard to match with the ssh+nfs cross test setup).
Ok.
> currently the test system depends on perl (for header conformance
> tests and memory leak tests) which is already a pain.
Ok.
> python is another hard to build dependency...
Granted. But... is this something you have actually tried, or are
you basing this on your experience with Perl? Some years ago I
worked on packaging a bunch of things (including Python and Perl)
for various free and proprietary systems and Perl was always the
time-sink. Funnily enough it was there to support a test suite
then too...
> ...and you have to make sure that python language changes don't
> affect the behaviour.
For sure. Both I8C and I8X have pretty comprehensive test suites.
I8C has tests with 98% coverage so I'm pretty sure it's doing what
it should. I test every commit on Pythons 2.[67], 3.[1-5] and the
latest development and nightly snapshots. I8X has only 87% coverage
but I'm working on that. I'd like both to be 100% by the time I
have a full GLIBC series ready to commit.
> i think this is not a critical issue since the build can succeed
> without i8*, but it is worth noting.
Yes, if the tools aren't installed the build will proceed as normal,
you just won't get the extra stuff this series adds.
Thanks,
Gary
Adhemerval Zanella wrote:
> I would prefer something related to systemtap way
How do you mean?
> I would prefer something more decoupled of GLIBC, so if we need to
> get rid off its support if a new strategy/project shows a superior
> approach it would be easier.
Decoupled how?
> I would also prefer to no be dependent of external interpreters to
> implement such compiler/instrumentation (since IMHO there is no
> strong reason to depend of python or any other interpreter for such
> project).
>
> But also this I also see it is not a strong reason to really oppose
> it.
Ok. Thank you :)
Cheers,
Gary
On 22-10-2015 11:40, Gary Benson wrote:
> Adhemerval Zanella wrote:
>> I would prefer something related to systemtap way
>
> How do you mean?
>
>> I would prefer something more decoupled of GLIBC, so if we need to
>> get rid off its support if a new strategy/project shows a superior
>> approach it would be easier.
>
> Decoupled how?
Good question, I do not have the answer. My idea is to just to be bounded
to carry any debugging mechanism forever over releases and have the freedom
to rework and remove any of them if it is suitable.
>
>> I would also prefer to no be dependent of external interpreters to
>> implement such compiler/instrumentation (since IMHO there is no
>> strong reason to depend of python or any other interpreter for such
>> project).
>>
>> But also this I also see it is not a strong reason to really oppose
>> it.
>
> Ok. Thank you :)
>
> Cheers,
> Gary
>
Adhemerval Zanella wrote:
> On 22-10-2015 11:40, Gary Benson wrote:
> > Adhemerval Zanella wrote:
> > > I would prefer something more decoupled of GLIBC, so if we need
> > > to get rid off its support if a new strategy/project shows a
> > > superior approach it would be easier.
> >
> > Decoupled how?
>
> Good question, I do not have the answer. My idea is to just to be
> bounded to carry any debugging mechanism forever over releases and
> have the freedom to rework and remove any of them if it is suitable.
Ah, I see.
I'm hoping I've built enough flexibility into the note format [1]
that any future problems we need to solve can be handled by
refinements rather than replacement. Ultimately that's something
only time can decide, but I'd appreciate any comments people have
on that spec.
Cheers,
Gary
On Wed, 21 Oct 2015, Adhemerval Zanella wrote:
> This can be done with proper documentation and in the release cycle: the
> release maintainer will recreate the assembly annotations for the
> release architectures. My only concern is the kernel notes dependency,
> however even those we need to take care since you also do not know prior
> hand which version you will end using.
>
> Also we can use the same strategy with the ulp updates for link testing.
There are lots of ABI variations that might be relevant to this; requiring
updates for all of those at release time doesn't seem a good idea (we
rarely manager to get a full set of test reports for a release). Lots
more variations than applicable for ulps updates (and really we should be
able to reduce the number of libm-test-ulps files further; given a generic
one for (flt-32, dbl-64) and one for (flt-32, dbl-64, ldbl-128), only
i386, x86_64, ia64, m68k and powerpc ought to need their own versions,
plus maybe hppa and sh because of underlying floating-point issues, though
maybe fused operations would slightly increase that number).
@@ -138,6 +138,12 @@ libpthread-routines = nptl-init vars events version pt-interp \
# pthread_setgid pthread_setegid pthread_setregid \
# pthread_setresgid
+ifneq ($(I8C),no)
+infinity-routines = infinity-map_lwp2thr
+
+libpthread-routines += $(infinity-routines)
+endif
+
libpthread-shared-only-routines = version pt-interp pt-allocrtsig \
unwind-forcedunwind
libpthread-static-only-routines = pthread_atfork
@@ -414,6 +420,26 @@ ifneq ($(have-cxx-thread_local),yes)
tests-unsupported += tst-thread_local1
endif
+tests += tst-infinity
+ifneq ($(I8C),no)
+ifneq ($(I8X),no)
+infinity-tests = $(addprefix tst-,$(addsuffix .py,$(infinity-routines)))
+
+$(objpfx)tst-infinity.out: $(objpfx)libpthread.so$(libpthread.so-version) \
+ $(infinity-tests)
+ $(I8X) -i $< -I$(common-objpfx) $(infinity-tests) > $@; \
+ $(evaluate-test)
+
+tests-special += $(objpfx)tst-infinity.out
+else
+# I8X is not installed; the tests cannot be run.
+tests-unsupported += tst-infinity
+endif
+else
+# I8C is not installed; libpthread.so will not contain notes.
+tests-unsupported += tst-infinity
+endif
+
include ../Rules
ifeq (yes,$(build-shared))
new file mode 100644
@@ -0,0 +1,61 @@
+/* Which thread is running on an LWP?
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+
+/* Given an lwpid_t identifying an LWP, return TD_OK and the
+ descriptor of the thread running on it, or a non-TD_OK
+ td_err_e code indicating the reason for failure and an
+ undefined value that must be ignored. The caller must
+ ensure that __pthread_initialize_minimal has gotten far
+ enough; see the comments in infinity_map_lwp2thr.i8 for
+ specifics. */
+
+define libpthread::__lookup_th_unique returns td_err_e, ptr
+ argument lwpid_t lwpid
+ extern ps_get_ta_f procservice::get_thread_area
+
+ // stack 0: procservice::get_thread_area
+ // stack 1: lwpid
+ load I8_TS_CTA_VALUE
+ // stack 0: I8_TS_CTA_VALUE
+ // stack 1: procservice::get_thread_area
+ // stack 2: lwpid
+ swap
+ // stack 0: procservice::get_thread_area
+ // stack 1: I8_TS_CTA_VALUE
+ // stack 2: lwpid
+ call
+ // stack 0: PS_OK (hopefully) or something else
+ // stack 1: the thing we want (hopefully) or junk
+ load PS_OK
+ // stack 0: PS_OK
+ // stack 1: PS_OK (hopefully) or something else
+ // stack 2: the thing we want (hopefully) or junk
+ bne get_thread_area_failed
+ // stack 0: the thing we want
+ load TD_OK
+ // stack 0: TD_OK
+ // stack 1: the thing we want
+ return
+
+get_thread_area_failed:
+ // stack 0: junk
+ load TD_ERR
+ // stack 0: TD_OK
+ // stack 1: junk
+ return
new file mode 100644
@@ -0,0 +1,76 @@
+/* Which thread is running on an LWP?
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+
+/* Given an lwpid_t identifying an LWP, return TD_OK and the
+ descriptor of the thread running on it, or a non-TD_OK
+ td_err_e code indicating the reason for failure and an
+ undefined value that must be ignored. The caller must
+ ensure that __pthread_initialize_minimal has gotten far
+ enough; see the comments in infinity_map_lwp2thr.i8 for
+ specifics. */
+
+define libpthread::__lookup_th_unique returns td_err_e, ptr
+ argument lwpid_t lwpid
+ extern ps_getreg_f procservice::get_register
+
+ // stack 0: procservice::get_register
+ // stack 1: lwpid
+ load I8_TS_REG_OFFSET
+ // stack 0: I8_TS_REG_OFFSET
+ // stack 1: procservice::get_thread_area
+ // stack 2: lwpid
+ swap
+ // stack 0: procservice::get_thread_area
+ // stack 1: I8_TS_REG_OFFSET
+ // stack 2: lwpid
+ load I8_TS_REG_SIZE
+ // stack 0: I8_TS_REG_SIZE
+ // stack 1: procservice::get_thread_area
+ // stack 2: I8_TS_REG_OFFSET
+ // stack 3: lwpid
+ swap
+ // stack 1: procservice::get_thread_area
+ // stack 0: I8_TS_REG_SIZE
+ // stack 2: I8_TS_REG_OFFSET
+ // stack 3: lwpid
+ call
+ // stack 0: PS_OK (hopefully) or something else
+ // stack 1: contents of register (hopefully) or junk
+ load PS_OK
+ // stack 0: PS_OK
+ // stack 1: PS_OK (hopefully) or something else
+ // stack 2: contents of register (hopefully) or junk
+ bne get_register_failed
+ // stack 0: contents of register
+ load I8_TS_REG_BIAS
+ // stack 0: I8_TS_REG_BIAS
+ // stack 1: contents of register
+ add
+ // stack 0: biased contents of register
+ load TD_OK
+ // stack 0: TD_OK
+ // stack 1: biased contents of register
+ return
+
+get_register_failed:
+ // stack 0: junk
+ load TD_ERR
+ // stack 0: TD_OK
+ // stack 1: junk
+ return
new file mode 100644
@@ -0,0 +1,140 @@
+/* Which thread is running on an LWP?
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+
+/* Given an lwpid_t identifying an LWP, return TD_OK and the
+ descriptor of the thread running on it, or a non-TD_OK
+ td_err_e code indicating the reason for failure and an
+ undefined value that must be ignored. The caller must
+ ensure that __pthread_initialize_minimal has gotten far
+ enough; see the comments in infinity_map_lwp2thr.i8 for
+ specifics. */
+
+define libpthread::__lookup_th_unique returns td_err_e, ptr
+ argument lwpid_t lwpid
+ extern ps_getreg_f procservice::get_register
+ extern ps_get_ta_f procservice::get_thread_area
+
+ // stack 0: procservice::get_thread_area
+ // stack 1: procservice::get_reg
+ // stack 2: lwpid
+ load lwpid
+ // stack 0: lwpid
+ // stack 1: procservice::get_thread_area
+ // stack 2: procservice::get_reg
+ // stack 3: lwpid
+ rot
+ // stack 0: procservice::get_thread_area
+ // stack 1: procservice::get_reg
+ // stack 2: lwpid
+ // stack 3: lwpid
+ rot
+ // stack 0: procservice::get_reg
+ // stack 1: lwpid
+ // stack 2: procservice::get_thread_area
+ // stack 3: lwpid
+ load I8_TS_RTA_OFFSET
+ // stack 0: I8_TS_RTA_OFFSET
+ // stack 1: procservice::get_thread_area
+ // stack 2: lwpid
+ // stack 3: procservice::get_thread_area
+ // stack 4: lwpid
+ swap
+ // stack 0: procservice::get_thread_area
+ // stack 1: I8_TS_RTA_OFFSET
+ // stack 2: lwpid
+ // stack 3: procservice::get_thread_area
+ // stack 4: lwpid
+ load I8_TS_RTA_SIZE
+ // stack 0: I8_TS_RTA_SIZE
+ // stack 1: procservice::get_thread_area
+ // stack 2: I8_TS_RTA_OFFSET
+ // stack 3: lwpid
+ // stack 4: procservice::get_thread_area
+ // stack 5: lwpid
+ swap
+ // stack 0: procservice::get_thread_area
+ // stack 1: I8_TS_RTA_SIZE
+ // stack 2: I8_TS_RTA_OFFSET
+ // stack 3: lwpid
+ // stack 4: procservice::get_thread_area
+ // stack 5: lwpid
+ call
+ // stack 0: PS_OK (hopefully) or something else
+ // stack 1: contents of register (hopefully) or junk
+ // stack 2: procservice::get_thread_area
+ // stack 3: lwpid
+ load PS_OK
+ // stack 0: PS_OK
+ // stack 1: PS_OK (hopefully) or something else
+ // stack 2: contents of register (hopefully) or junk
+ // stack 3: procservice::get_thread_area
+ // stack 4: lwpid
+ bne get_register_failed
+ // stack 0: contents of register
+ // stack 1: procservice::get_thread_area
+ // stack 2: lwpid
+ cast 0 int
+ // stack 0: I8_TS_RTA_SCALE
+ // stack 1: contents of register
+ // stack 2: procservice::get_thread_area
+ // stack 3: lwpid
+ load I8_TS_RTA_SCALE
+ // stack 0: I8_TS_RTA_SCALE
+ // stack 1: contents of register
+ // stack 2: procservice::get_thread_area
+ // stack 3: lwpid
+ shr
+ // stack 0: scaled contents of register
+ // stack 1: procservice::get_thread_area
+ // stack 2: lwpid
+ swap
+ // stack 0: procservice::get_thread_area
+ // stack 1: scaled contents of register
+ // stack 2: lwpid
+ call
+ // stack 0: PS_OK (hopefully) or something else
+ // stack 1: the thing we want (hopefully) or junk
+ load PS_OK
+ // stack 0: PS_OK
+ // stack 1: PS_OK (hopefully) or something else
+ // stack 2: the thing we want (hopefully) or junk
+ bne get_thread_area_failed
+ // stack 0: the thing we want
+ load TD_OK
+ // stack 0: TD_OK
+ // stack 1: the thing we want
+ return
+
+get_register_failed:
+ // stack 0: junk
+ // stack 1: procservice::get_thread_area
+ // stack 2: lwpid
+ load TD_ERR
+ // stack 0: TD_OK
+ // stack 1: junk
+ // stack 2: procservice::get_thread_area
+ // stack 3: lwpid
+ return
+
+get_thread_area_failed:
+ // stack 0: junk
+ load TD_ERR
+ // stack 0: TD_ERR
+ // stack 1: junk
+ return
new file mode 100644
@@ -0,0 +1,121 @@
+/* Which thread is running on an LWP?
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include "infinity-nptl.i8"
+#include "infinity-thread_self.h"
+
+#if I8_THREAD_SELF == I8_TS_CONST_THREAD_AREA
+# include "infinity-lookup_th_unique_cta.i8"
+#endif
+#if I8_THREAD_SELF == I8_TS_REGISTER
+# include "infinity-lookup_th_unique_reg.i8"
+#endif
+#if I8_THREAD_SELF == I8_TS_REGISTER_THREAD_AREA
+# include "infinity-lookup_th_unique_rta.i8"
+#endif
+
+/* Given an lwpid_t identifying an LWP, return TD_OK and the
+ descriptor of the thread running on it, or a non-TD_OK
+ td_err_e code indicating the reason for failure and an
+ undefined value that must be ignored. Thread descriptors
+ are opaque pointers and should not be dereferenced outside
+ of this library. */
+
+define libpthread::map_lwp2thr returns td_err_e, ptr
+ argument lwpid_t lwpid
+ extern func td_err_e, ptr (lwpid_t) __lookup_th_unique
+ extern func pid_t () i8core::getpid
+ extern ptr __stack_user
+
+ /* We cannot rely on thread registers and such information at all
+ before __pthread_initialize_minimal has gotten far enough: they
+ sometimes contain garbage left by the kernel at exec that would
+ confuse us. If it looks like initialization is incomplete we
+ fake a special descriptor of NULL to indicate the initial thread.
+ Other routines in this library recognise this special descriptor
+ and act accordingly. */
+
+ // stack 0: __stack_user
+ // stack 1: i8core::getpid
+ // stack 2: __lookup_th_unique
+ // stack 3: lwpid
+ load LIST_T_NEXT_OFFSET
+ // stack 0: LIST_T_NEXT_OFFSET
+ // stack 1: __stack_user
+ // stack 2: i8core::getpid
+ // stack 3: __lookup_th_unique
+ // stack 4: lwpid
+ add
+ // stack 0: __stack_user + LIST_T_NEXT_OFFSET
+ // stack 1: i8core::getpid
+ // stack 2: __lookup_th_unique
+ // stack 3: lwpid
+ deref ptr
+ // stack 0: __stack_user->next
+ // stack 1: i8core::getpid
+ // stack 2: __lookup_th_unique
+ // stack 3: lwpid
+ load NULL
+ bne libpthread_is_initialized
+
+initialization_is_incomplete:
+ // stack 0: i8core::getpid
+ // stack 1: __lookup_th_unique
+ // stack 2: lwpid
+ swap
+ // stack 0: __lookup_th_unique
+ // stack 1: i8core::getpid
+ // stack 2: lwpid
+ drop
+ // stack 0: i8core::getpid
+ // stack 1: lwpid
+ load NULL
+ // stack 0: NULL
+ // stack 1: i8core::getpid
+ // stack 2: lwpid
+ rot
+ // stack 0: i8core::getpid
+ // stack 1: lwpid
+ // stack 2: NULL
+ call
+ // stack 0: main pid
+ // stack 1: lwpid
+ // stack 2: NULL
+ beq is_main_thread
+
+not_main_thread:
+ // stack 0: NULL
+ load TD_ERR
+ return
+
+is_main_thread:
+ // stack 0: NULL
+ load TD_OK
+ return
+
+libpthread_is_initialized:
+ // stack 0: i8core::getpid
+ // stack 1: __lookup_th_unique
+ // stack 2: lwpid
+ drop
+ // stack 0: __lookup_th_unique
+ // stack 1: lwpid
+ call
+ // stack 0: TD_OK (or not)
+ // stack 1: descriptor (or junk)
+ return
new file mode 100644
@@ -0,0 +1,36 @@
+/* Common definitions for NPTL Infinity functions.
+ Copyright (C) 2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* libpthread and libthread_db generated constants. */
+#include "infinity-nptl-constants.h"
+#include "infinity-nptl_db-constants.h"
+
+/* XXX. */
+typedef int size_t
+
+/* XXX. */
+typedef s32 pid_t
+typedef pid_t lwpid_t
+
+/* XXX. */
+typedef s32 td_err_e
+typedef s32 ps_err_e
+
+/* XXX. */
+typedef func ps_err_e, ptr (lwpid_t, int, int) ps_getreg_f
+typedef func ps_err_e, ptr (lwpid_t, int) ps_get_ta_f
new file mode 100644
@@ -0,0 +1,155 @@
+from i8c.runtime import TestCase
+import struct
+
+TestCase.import_builtin_constants()
+TestCase.import_constants_from("infinity-nptl-constants.h")
+TestCase.import_constants_from("infinity-nptl_db-constants.h")
+
+class TestMapLwp2Thr(TestCase):
+ TESTFUNC = "libpthread::map_lwp2thr(i)ip"
+ MAIN_PID = 30000
+
+ def setUp(self):
+ # Create flags
+ self.ps_get_register_called = False
+ self.ps_get_thread_area_called = False
+ # Store the address of __stack_user
+ note = self.i8ctx.get_function(self.TESTFUNC)
+ symbols = note.external_pointers
+ self.assertEqual(len(symbols), 1)
+ self.stack_user_p = symbols[0]
+
+ def read_memory(self, fmt, addr):
+ # The only dereference we do is __stack_user.next
+ self.assertEqual(addr, self.stack_user_p + LIST_T_NEXT_OFFSET)
+ return struct.pack(fmt, self.STACK_USER_NEXT)
+
+ def call_i8core_getpid(self):
+ """Implementation of i8core::getpid."""
+ return self.MAIN_PID
+
+ def call_procservice_get_register(self, lwpid, offset, size):
+ """Implementation of procservice::get_register."""
+ self.assertFalse(self.ps_get_register_called)
+ result = getattr(self, "PS_GETREG_RESULT", None)
+ if result is None:
+ self.fail("unexpected ps_get_register")
+ self.assertEqual(lwpid, self.lwpid)
+ self.assertNotEqual(offset, self.lwpid)
+ self.assertGreaterEqual(offset, 0)
+ # We can't really say much about offset. It's an offset into
+ # a prgregset_t structure, so it's probably not huge and it's
+ # probably aligned to the machine's wordsize.
+ self.assertLess(offset, 128 * 8) # =128 64-bit registers (IA-64)
+ bytes_per_word, check = divmod(self.i8ctx.wordsize, 8)
+ self.assertNotEqual(bytes_per_word, 0)
+ self.assertEqual(check, 0)
+ self.assertEqual(offset % bytes_per_word, 0)
+ self.assertIn(size, (8, 16, 32, 64))
+ self.assertLessEqual(size, self.i8ctx.wordsize)
+ self.ps_get_register_called = True
+ return result
+
+ def call_procservice_get_thread_area(self, lwpid, idx):
+ """Implementation of procservice::get_thread_area."""
+ self.assertFalse(self.ps_get_thread_area_called)
+ result = getattr(self, "PS_GET_TA_RESULT", None)
+ if result is None:
+ self.fail("unexpected ps_get_thread_area")
+ self.assertEqual(lwpid, self.lwpid)
+ self.assertNotEqual(idx, self.lwpid)
+ self.ps_get_thread_area_called = True
+ return result
+
+ def check_I8_TS_CONST_THREAD_AREA_result(self, result):
+ # The result is whatever ps_get_thread_area returned
+ self.assertTrue(self.ps_get_thread_area_called)
+ self.assertEqual(result[0], TD_OK)
+ self.assertNotEqual(result[1], 0)
+ self.assertEqual(result[1], self.PS_GET_TA_RESULT[1])
+
+ def check_I8_TS_REGISTER_result(self, result):
+ # The result is what ps_get_register returned with some
+ # bias added. We'll assume the bias is fairly small.
+ self.assertTrue(self.ps_get_register_called)
+ self.assertEqual(result[0], TD_OK)
+ self.assertNotEqual(result[1], 0)
+ bias = result[1] - self.PS_GETREG_RESULT[1]
+ self.assertLess(abs(bias), 16384)
+
+ def check_I8_TS_REGISTER_THREAD_AREA_result(self, result):
+ # The result is whatever ps_get_thread_area returned
+ self.assertTrue(self.ps_get_register_called)
+ self.assertTrue(self.ps_get_thread_area_called)
+ self.assertEqual(result[0], TD_OK)
+ self.assertNotEqual(result[1], 0)
+ self.assertEqual(result[1], self.PS_GET_TA_RESULT[1])
+
+class TestMapLwp2Thr_uninit(TestMapLwp2Thr):
+ STACK_USER_NEXT = NULL
+
+ def test_map_lwp2thr(self):
+ """map_lwp2thr (nptl uninitialized, lwpid == main PID)"""
+ result = self.i8ctx.call(self.TESTFUNC, self.MAIN_PID)
+ self.assertEqual(len(result), 2)
+ self.assertEqual(result[0], TD_OK)
+ self.assertEqual(result[1], NULL)
+
+class TestMapLwp2Thr_uninit_wrongpid(TestMapLwp2Thr):
+ STACK_USER_NEXT = NULL
+
+ def test_map_lwp2thr(self):
+ """map_lwp2thr (nptl uninitialized, lwpid != main PID)"""
+ result = self.i8ctx.call(self.TESTFUNC, self.MAIN_PID + 1)
+ self.assertEqual(len(result), 2)
+ self.assertEqual(result[0], TD_ERR)
+
+class TestMapLwp2Thr_init_getreg_fail(TestMapLwp2Thr):
+ STACK_USER_NEXT = 0x1fff
+ PS_GETREG_RESULT = PS_ERR, 0x23ff00fa
+ PS_GET_TA_RESULT = PS_OK, 0x89ab1234
+
+ def test_map_lwp2thr(self):
+ """map_lwp2thr (nptl initialized, ps_get_register fails)"""
+ self.lwpid = self.MAIN_PID + 1
+ result = self.i8ctx.call(self.TESTFUNC, self.lwpid)
+ self.assertEqual(len(result), 2)
+ if self.ps_get_register_called:
+ self.assertEqual(result[0], TD_ERR)
+ else:
+ # This failure isn't a problem for this platform
+ self.check_I8_TS_CONST_THREAD_AREA_result(result)
+
+class TestMapLwp2Thr_init_gta_fail(TestMapLwp2Thr):
+ STACK_USER_NEXT = 0x1fff
+ PS_GETREG_RESULT = PS_OK, 0x23ff00fa
+ PS_GET_TA_RESULT = PS_ERR, 0x89ab1234
+
+ def test_map_lwp2thr(self):
+ """map_lwp2thr (nptl initialized, ps_get_thread_area fails)"""
+ self.lwpid = self.MAIN_PID + 1
+ result = self.i8ctx.call(self.TESTFUNC, self.lwpid)
+ self.assertEqual(len(result), 2)
+ if self.ps_get_thread_area_called:
+ self.assertEqual(result[0], TD_ERR)
+ else:
+ # This failure isn't a problem for this platform
+ self.check_I8_TS_REGISTER_result(result)
+
+class TestMapLwp2Thr_init_gta_ok(TestMapLwp2Thr):
+ STACK_USER_NEXT = 0x1fff
+ PS_GETREG_RESULT = PS_OK, 0x23ff00fa
+ PS_GET_TA_RESULT = PS_OK, 0x89ab1234
+
+ def test_map_lwp2thr(self):
+ """map_lwp2thr (nptl initialized, everything worked)"""
+ self.lwpid = self.MAIN_PID + 1
+ result = self.i8ctx.call(self.TESTFUNC, self.lwpid)
+ self.assertEqual(len(result), 2)
+ if self.ps_get_thread_area_called:
+ if self.ps_get_register_called:
+ self.check_I8_TS_REGISTER_THREAD_AREA_result(result)
+ else:
+ self.check_I8_TS_CONST_THREAD_AREA_result(result)
+ else:
+ self.check_I8_TS_REGISTER_result(result)