[RFC] gcc: Improve reproducability when building cc1/cc1plus
Commit Message
cc1/cc1plus both include a checksum of the object files, archives and
options used during linking. Unless the host binutils has been built
with --enable-deterministic-archives, the archives will have different
checksums each build due to changes in timestamps of the containing
object files, and thus the checksum that is embedded in cc1/cc1plus will
also change.
Fix this by passing "D" to ar/ranlib when creating the archives.
Signed-off-by: Jacob Kroon <jacob.kroon@gmail.com>
---
Makefile.in | 4 ++--
gcc/Makefile.in | 2 +-
libbacktrace/configure | 2 +-
libcody/Makefile.in | 4 ++--
libcpp/Makefile.in | 4 ++--
libdecnumber/Makefile.in | 4 ++--
libiberty/Makefile.in | 6 +++---
7 files changed, 13 insertions(+), 13 deletions(-)
Comments
On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> cc1/cc1plus both include a checksum of the object files, archives and
> options used during linking. Unless the host binutils has been built
> with --enable-deterministic-archives, the archives will have different
> checksums each build due to changes in timestamps of the containing
> object files, and thus the checksum that is embedded in cc1/cc1plus will
> also change.
>
> Fix this by passing "D" to ar/ranlib when creating the archives.
How portable is the D option? That does it work with Mac OS X's ar;
what about AIX's ar; what about Solaris's ar, etc?
GNU binutils is not the only ar which is supported for the host.
Thanks,
Andrew Pinski
>
> Signed-off-by: Jacob Kroon <jacob.kroon@gmail.com>
> ---
> Makefile.in | 4 ++--
> gcc/Makefile.in | 2 +-
> libbacktrace/configure | 2 +-
> libcody/Makefile.in | 4 ++--
> libcpp/Makefile.in | 4 ++--
> libdecnumber/Makefile.in | 4 ++--
> libiberty/Makefile.in | 6 +++---
> 7 files changed, 13 insertions(+), 13 deletions(-)
>
> diff --git a/Makefile.in b/Makefile.in
> index 860cf8f067b..8c451ed2121 100644
> --- a/Makefile.in
> +++ b/Makefile.in
> @@ -416,7 +416,7 @@ MAKEINFOFLAGS = --split-size=5000000
>
> AS = @AS@
> AR = @AR@
> -AR_FLAGS = rc
> +AR_FLAGS = rcD
> CC = @CC@
> CXX = @CXX@
> DLLTOOL = @DLLTOOL@
> @@ -23571,7 +23571,7 @@ configure-libbacktrace:
> $$s/$$module_srcdir/configure \
> --srcdir=$${topdir}/$$module_srcdir \
> $(HOST_CONFIGARGS) --build=${build_alias} --host=${host_alias} \
> - --target=${target_alias} \
> + --target=${target_alias} RANLIB="ranlib -D" \
> || exit 1
> @endif libbacktrace
>
> diff --git a/gcc/Makefile.in b/gcc/Makefile.in
> index 571e9c28e29..828b13d9b54 100644
> --- a/gcc/Makefile.in
> +++ b/gcc/Makefile.in
> @@ -243,7 +243,7 @@ AR = @AR@
> AR_FLAGS = rc
> NM = @NM@
> RANLIB = @RANLIB@
> -RANLIB_FLAGS = @ranlib_flags@
> +RANLIB_FLAGS = @ranlib_flags@ -D
>
> # Libraries to use on the host.
> HOST_LIBS = @HOST_LIBS@
> diff --git a/libbacktrace/configure b/libbacktrace/configure
> index a2f33c0f35d..5b18966ea80 100755
> --- a/libbacktrace/configure
> +++ b/libbacktrace/configure
> @@ -6669,7 +6669,7 @@ else
> fi
>
> test -z "$AR" && AR=ar
> -test -z "$AR_FLAGS" && AR_FLAGS=cru
> +test -z "$AR_FLAGS" && AR_FLAGS=crD
>
>
>
> diff --git a/libcody/Makefile.in b/libcody/Makefile.in
> index 7eaf8ace8ce..f1e8be352b5 100644
> --- a/libcody/Makefile.in
> +++ b/libcody/Makefile.in
> @@ -126,8 +126,8 @@ LIBCODY.O := buffer.o client.o fatal.o netclient.o netserver.o \
> all:: libcody.a
>
> libcody.a: $(LIBCODY.O)
> - $(AR) -cr $@ $^
> - $(RANLIB) $@
> + $(AR) -crD $@ $^
> + $(RANLIB) -D $@
>
> mostlyclean::
>
> diff --git a/libcpp/Makefile.in b/libcpp/Makefile.in
> index 34e4206957d..53d220c4e0b 100644
> --- a/libcpp/Makefile.in
> +++ b/libcpp/Makefile.in
> @@ -26,7 +26,7 @@ top_builddir = .
> VPATH = @srcdir@
> INSTALL = @INSTALL@
> AR = @AR@
> -ARFLAGS = cru
> +ARFLAGS = crD
> ACLOCAL = @ACLOCAL@
> AUTOCONF = @AUTOCONF@
> AUTOHEADER = @AUTOHEADER@
> @@ -101,7 +101,7 @@ all: libcpp.a $(USED_CATALOGS)
> libcpp.a: $(libcpp_a_OBJS)
> -rm -f libcpp.a
> $(AR) $(ARFLAGS) libcpp.a $(libcpp_a_OBJS)
> - $(RANLIB) libcpp.a
> + $(RANLIB) -D libcpp.a
>
> # Rules to rebuild the configuration
>
> diff --git a/libdecnumber/Makefile.in b/libdecnumber/Makefile.in
> index 7e086fccafa..3b03064fc4d 100644
> --- a/libdecnumber/Makefile.in
> +++ b/libdecnumber/Makefile.in
> @@ -26,7 +26,7 @@ top_builddir = .
> VPATH = @srcdir@
> INSTALL = @INSTALL@
> AR = @AR@
> -ARFLAGS = cru
> +ARFLAGS = crD
> ACLOCAL = @ACLOCAL@
> AUTOCONF = @AUTOCONF@
> AUTOHEADER = @AUTOHEADER@
> @@ -88,7 +88,7 @@ all: libdecnumber.a
> libdecnumber.a: $(libdecnumber_a_OBJS)
> -rm -f $@
> $(AR) $(ARFLAGS) $@ $(libdecnumber_a_OBJS)
> - $(RANLIB) $@
> + $(RANLIB) -D $@
>
> # Rules to rebuild the configuration
>
> diff --git a/libiberty/Makefile.in b/libiberty/Makefile.in
> index 884cc6c4af2..9f507a54657 100644
> --- a/libiberty/Makefile.in
> +++ b/libiberty/Makefile.in
> @@ -252,19 +252,19 @@ $(TARGETLIB): $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
> -rm -f $(TARGETLIB) pic/$(TARGETLIB) noasan/$(TARGETLIB)
> $(AR) $(AR_FLAGS) $(TARGETLIB) \
> $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
> - $(RANLIB) $(TARGETLIB)
> + $(RANLIB) -D $(TARGETLIB)
> if [ x"$(PICFLAG)" != x ]; then \
> cd pic; \
> $(AR) $(AR_FLAGS) $(TARGETLIB) \
> $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
> - $(RANLIB) $(TARGETLIB); \
> + $(RANLIB) -D $(TARGETLIB); \
> cd ..; \
> else true; fi; \
> if [ x"$(NOASANFLAG)" != x ]; then \
> cd noasan; \
> $(AR) $(AR_FLAGS) $(TARGETLIB) \
> $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
> - $(RANLIB) $(TARGETLIB); \
> + $(RANLIB) -D $(TARGETLIB); \
> cd ..; \
> else true; fi
>
On Sat, Nov 20, 2021 at 12:24:21AM -0800, Andrew Pinski via Gcc-patches wrote:
> On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
> >
> > cc1/cc1plus both include a checksum of the object files, archives and
> > options used during linking. Unless the host binutils has been built
> > with --enable-deterministic-archives, the archives will have different
> > checksums each build due to changes in timestamps of the containing
> > object files, and thus the checksum that is embedded in cc1/cc1plus will
> > also change.
> >
> > Fix this by passing "D" to ar/ranlib when creating the archives.
>
> How portable is the D option? That does it work with Mac OS X's ar;
> what about AIX's ar; what about Solaris's ar, etc?
> GNU binutils is not the only ar which is supported for the host.
It isn't portable and even if ar/ranlib do support that option, not all
users will want it, so forcing it upon them unconditionally is IMO not a
good idea. If anything, configure needs to check if those options are
supported and there needs to be preferrably non-default gcc configure
option to request it.
Jakub
On 11/20/21 09:32, Jakub Jelinek wrote:
> On Sat, Nov 20, 2021 at 12:24:21AM -0800, Andrew Pinski via Gcc-patches wrote:
>> On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
>> <gcc-patches@gcc.gnu.org> wrote:
>>>
>>> cc1/cc1plus both include a checksum of the object files, archives and
>>> options used during linking. Unless the host binutils has been built
>>> with --enable-deterministic-archives, the archives will have different
>>> checksums each build due to changes in timestamps of the containing
>>> object files, and thus the checksum that is embedded in cc1/cc1plus will
>>> also change.
>>>
>>> Fix this by passing "D" to ar/ranlib when creating the archives.
>>
>> How portable is the D option? That does it work with Mac OS X's ar;
>> what about AIX's ar; what about Solaris's ar, etc?
>> GNU binutils is not the only ar which is supported for the host.
>
> It isn't portable and even if ar/ranlib do support that option, not all
> users will want it, so forcing it upon them unconditionally is IMO not a
> good idea. If anything, configure needs to check if those options are
> supported and there needs to be preferrably non-default gcc configure
> option to request it.
>
Aha, I see. Then perhaps an easier and more portable solution for
improving the reproducibility would be to make it optional whether or
not to include the archives when calculating the checksums.
Or perhaps by checksumming the containing object files instead of the
archives themselves ?
Would something like that be acceptable ?
Jacob
On Sat, Nov 20, 2021 at 9:43 AM Jacob Kroon via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> On 11/20/21 09:32, Jakub Jelinek wrote:
> > On Sat, Nov 20, 2021 at 12:24:21AM -0800, Andrew Pinski via Gcc-patches wrote:
> >> On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
> >> <gcc-patches@gcc.gnu.org> wrote:
> >>>
> >>> cc1/cc1plus both include a checksum of the object files, archives and
> >>> options used during linking. Unless the host binutils has been built
> >>> with --enable-deterministic-archives, the archives will have different
> >>> checksums each build due to changes in timestamps of the containing
> >>> object files, and thus the checksum that is embedded in cc1/cc1plus will
> >>> also change.
> >>>
> >>> Fix this by passing "D" to ar/ranlib when creating the archives.
> >>
> >> How portable is the D option? That does it work with Mac OS X's ar;
> >> what about AIX's ar; what about Solaris's ar, etc?
> >> GNU binutils is not the only ar which is supported for the host.
> >
> > It isn't portable and even if ar/ranlib do support that option, not all
> > users will want it, so forcing it upon them unconditionally is IMO not a
> > good idea. If anything, configure needs to check if those options are
> > supported and there needs to be preferrably non-default gcc configure
> > option to request it.
> >
>
> Aha, I see. Then perhaps an easier and more portable solution for
> improving the reproducibility would be to make it optional whether or
> not to include the archives when calculating the checksums.
>
> Or perhaps by checksumming the containing object files instead of the
> archives themselves ?
>
> Would something like that be acceptable ?
At SUSE we are carrying patches to not include the checksum at all
but instead use the linker generated build-id as checksum for PCH.
I suppose when we add the configure option to disable PCH support
we could also elide checksum processing (or as in the patch simply
always use all-zeros).
Attached for reference.
Richard.
>
> Jacob
@@ -416,7 +416,7 @@ MAKEINFOFLAGS = --split-size=5000000
AS = @AS@
AR = @AR@
-AR_FLAGS = rc
+AR_FLAGS = rcD
CC = @CC@
CXX = @CXX@
DLLTOOL = @DLLTOOL@
@@ -23571,7 +23571,7 @@ configure-libbacktrace:
$$s/$$module_srcdir/configure \
--srcdir=$${topdir}/$$module_srcdir \
$(HOST_CONFIGARGS) --build=${build_alias} --host=${host_alias} \
- --target=${target_alias} \
+ --target=${target_alias} RANLIB="ranlib -D" \
|| exit 1
@endif libbacktrace
@@ -243,7 +243,7 @@ AR = @AR@
AR_FLAGS = rc
NM = @NM@
RANLIB = @RANLIB@
-RANLIB_FLAGS = @ranlib_flags@
+RANLIB_FLAGS = @ranlib_flags@ -D
# Libraries to use on the host.
HOST_LIBS = @HOST_LIBS@
@@ -6669,7 +6669,7 @@ else
fi
test -z "$AR" && AR=ar
-test -z "$AR_FLAGS" && AR_FLAGS=cru
+test -z "$AR_FLAGS" && AR_FLAGS=crD
@@ -126,8 +126,8 @@ LIBCODY.O := buffer.o client.o fatal.o netclient.o netserver.o \
all:: libcody.a
libcody.a: $(LIBCODY.O)
- $(AR) -cr $@ $^
- $(RANLIB) $@
+ $(AR) -crD $@ $^
+ $(RANLIB) -D $@
mostlyclean::
@@ -26,7 +26,7 @@ top_builddir = .
VPATH = @srcdir@
INSTALL = @INSTALL@
AR = @AR@
-ARFLAGS = cru
+ARFLAGS = crD
ACLOCAL = @ACLOCAL@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
@@ -101,7 +101,7 @@ all: libcpp.a $(USED_CATALOGS)
libcpp.a: $(libcpp_a_OBJS)
-rm -f libcpp.a
$(AR) $(ARFLAGS) libcpp.a $(libcpp_a_OBJS)
- $(RANLIB) libcpp.a
+ $(RANLIB) -D libcpp.a
# Rules to rebuild the configuration
@@ -26,7 +26,7 @@ top_builddir = .
VPATH = @srcdir@
INSTALL = @INSTALL@
AR = @AR@
-ARFLAGS = cru
+ARFLAGS = crD
ACLOCAL = @ACLOCAL@
AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
@@ -88,7 +88,7 @@ all: libdecnumber.a
libdecnumber.a: $(libdecnumber_a_OBJS)
-rm -f $@
$(AR) $(ARFLAGS) $@ $(libdecnumber_a_OBJS)
- $(RANLIB) $@
+ $(RANLIB) -D $@
# Rules to rebuild the configuration
@@ -252,19 +252,19 @@ $(TARGETLIB): $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
-rm -f $(TARGETLIB) pic/$(TARGETLIB) noasan/$(TARGETLIB)
$(AR) $(AR_FLAGS) $(TARGETLIB) \
$(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
- $(RANLIB) $(TARGETLIB)
+ $(RANLIB) -D $(TARGETLIB)
if [ x"$(PICFLAG)" != x ]; then \
cd pic; \
$(AR) $(AR_FLAGS) $(TARGETLIB) \
$(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
- $(RANLIB) $(TARGETLIB); \
+ $(RANLIB) -D $(TARGETLIB); \
cd ..; \
else true; fi; \
if [ x"$(NOASANFLAG)" != x ]; then \
cd noasan; \
$(AR) $(AR_FLAGS) $(TARGETLIB) \
$(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
- $(RANLIB) $(TARGETLIB); \
+ $(RANLIB) -D $(TARGETLIB); \
cd ..; \
else true; fi