diff mbox series

[RFC] gcc: Improve reproducability when building cc1/cc1plus

Message ID 20211120081748.1160013-1-jacob.kroon@gmail.com
State New
Headers show
Series [RFC] gcc: Improve reproducability when building cc1/cc1plus | expand

Commit Message

Jacob Kroon Nov. 20, 2021, 8:17 a.m. UTC
cc1/cc1plus both include a checksum of the object files, archives and
options used during linking. Unless the host binutils has been built
with --enable-deterministic-archives, the archives will have different
checksums each build due to changes in timestamps of the containing
object files, and thus the checksum that is embedded in cc1/cc1plus will
also change.

Fix this by passing "D" to ar/ranlib when creating the archives.

Signed-off-by: Jacob Kroon <jacob.kroon@gmail.com>
---
 Makefile.in              | 4 ++--
 gcc/Makefile.in          | 2 +-
 libbacktrace/configure   | 2 +-
 libcody/Makefile.in      | 4 ++--
 libcpp/Makefile.in       | 4 ++--
 libdecnumber/Makefile.in | 4 ++--
 libiberty/Makefile.in    | 6 +++---
 7 files changed, 13 insertions(+), 13 deletions(-)

Comments

Andrew Pinski Nov. 20, 2021, 8:24 a.m. UTC | #1
On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> cc1/cc1plus both include a checksum of the object files, archives and
> options used during linking. Unless the host binutils has been built
> with --enable-deterministic-archives, the archives will have different
> checksums each build due to changes in timestamps of the containing
> object files, and thus the checksum that is embedded in cc1/cc1plus will
> also change.
>
> Fix this by passing "D" to ar/ranlib when creating the archives.

How portable is the D option? That does it work with Mac OS X's ar;
what about AIX's ar; what about Solaris's ar, etc?
GNU binutils is not the only ar which is supported for the host.

Thanks,
Andrew Pinski


>
> Signed-off-by: Jacob Kroon <jacob.kroon@gmail.com>
> ---
>  Makefile.in              | 4 ++--
>  gcc/Makefile.in          | 2 +-
>  libbacktrace/configure   | 2 +-
>  libcody/Makefile.in      | 4 ++--
>  libcpp/Makefile.in       | 4 ++--
>  libdecnumber/Makefile.in | 4 ++--
>  libiberty/Makefile.in    | 6 +++---
>  7 files changed, 13 insertions(+), 13 deletions(-)
>
> diff --git a/Makefile.in b/Makefile.in
> index 860cf8f067b..8c451ed2121 100644
> --- a/Makefile.in
> +++ b/Makefile.in
> @@ -416,7 +416,7 @@ MAKEINFOFLAGS = --split-size=5000000
>
>  AS = @AS@
>  AR = @AR@
> -AR_FLAGS = rc
> +AR_FLAGS = rcD
>  CC = @CC@
>  CXX = @CXX@
>  DLLTOOL = @DLLTOOL@
> @@ -23571,7 +23571,7 @@ configure-libbacktrace:
>           $$s/$$module_srcdir/configure \
>           --srcdir=$${topdir}/$$module_srcdir \
>           $(HOST_CONFIGARGS) --build=${build_alias} --host=${host_alias} \
> -         --target=${target_alias}  \
> +         --target=${target_alias}  RANLIB="ranlib -D" \
>           || exit 1
>  @endif libbacktrace
>
> diff --git a/gcc/Makefile.in b/gcc/Makefile.in
> index 571e9c28e29..828b13d9b54 100644
> --- a/gcc/Makefile.in
> +++ b/gcc/Makefile.in
> @@ -243,7 +243,7 @@ AR = @AR@
>  AR_FLAGS = rc
>  NM = @NM@
>  RANLIB = @RANLIB@
> -RANLIB_FLAGS = @ranlib_flags@
> +RANLIB_FLAGS = @ranlib_flags@ -D
>
>  # Libraries to use on the host.
>  HOST_LIBS = @HOST_LIBS@
> diff --git a/libbacktrace/configure b/libbacktrace/configure
> index a2f33c0f35d..5b18966ea80 100755
> --- a/libbacktrace/configure
> +++ b/libbacktrace/configure
> @@ -6669,7 +6669,7 @@ else
>  fi
>
>  test -z "$AR" && AR=ar
> -test -z "$AR_FLAGS" && AR_FLAGS=cru
> +test -z "$AR_FLAGS" && AR_FLAGS=crD
>
>
>
> diff --git a/libcody/Makefile.in b/libcody/Makefile.in
> index 7eaf8ace8ce..f1e8be352b5 100644
> --- a/libcody/Makefile.in
> +++ b/libcody/Makefile.in
> @@ -126,8 +126,8 @@ LIBCODY.O := buffer.o client.o fatal.o netclient.o netserver.o \
>  all:: libcody.a
>
>  libcody.a: $(LIBCODY.O)
> -       $(AR) -cr $@ $^
> -       $(RANLIB) $@
> +       $(AR) -crD $@ $^
> +       $(RANLIB) -D $@
>
>  mostlyclean::
>
> diff --git a/libcpp/Makefile.in b/libcpp/Makefile.in
> index 34e4206957d..53d220c4e0b 100644
> --- a/libcpp/Makefile.in
> +++ b/libcpp/Makefile.in
> @@ -26,7 +26,7 @@ top_builddir = .
>  VPATH = @srcdir@
>  INSTALL = @INSTALL@
>  AR = @AR@
> -ARFLAGS = cru
> +ARFLAGS = crD
>  ACLOCAL = @ACLOCAL@
>  AUTOCONF = @AUTOCONF@
>  AUTOHEADER = @AUTOHEADER@
> @@ -101,7 +101,7 @@ all: libcpp.a $(USED_CATALOGS)
>  libcpp.a: $(libcpp_a_OBJS)
>         -rm -f libcpp.a
>         $(AR) $(ARFLAGS) libcpp.a $(libcpp_a_OBJS)
> -       $(RANLIB) libcpp.a
> +       $(RANLIB) -D libcpp.a
>
>  # Rules to rebuild the configuration
>
> diff --git a/libdecnumber/Makefile.in b/libdecnumber/Makefile.in
> index 7e086fccafa..3b03064fc4d 100644
> --- a/libdecnumber/Makefile.in
> +++ b/libdecnumber/Makefile.in
> @@ -26,7 +26,7 @@ top_builddir = .
>  VPATH = @srcdir@
>  INSTALL = @INSTALL@
>  AR = @AR@
> -ARFLAGS = cru
> +ARFLAGS = crD
>  ACLOCAL = @ACLOCAL@
>  AUTOCONF = @AUTOCONF@
>  AUTOHEADER = @AUTOHEADER@
> @@ -88,7 +88,7 @@ all: libdecnumber.a
>  libdecnumber.a: $(libdecnumber_a_OBJS)
>         -rm -f $@
>         $(AR) $(ARFLAGS) $@ $(libdecnumber_a_OBJS)
> -       $(RANLIB) $@
> +       $(RANLIB) -D $@
>
>  # Rules to rebuild the configuration
>
> diff --git a/libiberty/Makefile.in b/libiberty/Makefile.in
> index 884cc6c4af2..9f507a54657 100644
> --- a/libiberty/Makefile.in
> +++ b/libiberty/Makefile.in
> @@ -252,19 +252,19 @@ $(TARGETLIB): $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
>         -rm -f $(TARGETLIB) pic/$(TARGETLIB) noasan/$(TARGETLIB)
>         $(AR) $(AR_FLAGS) $(TARGETLIB) \
>           $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
> -       $(RANLIB) $(TARGETLIB)
> +       $(RANLIB) -D $(TARGETLIB)
>         if [ x"$(PICFLAG)" != x ]; then \
>           cd pic; \
>           $(AR) $(AR_FLAGS) $(TARGETLIB) \
>             $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
> -         $(RANLIB) $(TARGETLIB); \
> +         $(RANLIB) -D $(TARGETLIB); \
>           cd ..; \
>         else true; fi; \
>         if [ x"$(NOASANFLAG)" != x ]; then \
>           cd noasan; \
>           $(AR) $(AR_FLAGS) $(TARGETLIB) \
>             $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
> -         $(RANLIB) $(TARGETLIB); \
> +         $(RANLIB) -D $(TARGETLIB); \
>           cd ..; \
>         else true; fi
>
Jakub Jelinek Nov. 20, 2021, 8:32 a.m. UTC | #2
On Sat, Nov 20, 2021 at 12:24:21AM -0800, Andrew Pinski via Gcc-patches wrote:
> On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
> <gcc-patches@gcc.gnu.org> wrote:
> >
> > cc1/cc1plus both include a checksum of the object files, archives and
> > options used during linking. Unless the host binutils has been built
> > with --enable-deterministic-archives, the archives will have different
> > checksums each build due to changes in timestamps of the containing
> > object files, and thus the checksum that is embedded in cc1/cc1plus will
> > also change.
> >
> > Fix this by passing "D" to ar/ranlib when creating the archives.
> 
> How portable is the D option? That does it work with Mac OS X's ar;
> what about AIX's ar; what about Solaris's ar, etc?
> GNU binutils is not the only ar which is supported for the host.

It isn't portable and even if ar/ranlib do support that option, not all
users will want it, so forcing it upon them unconditionally is IMO not a
good idea.  If anything, configure needs to check if those options are
supported and there needs to be preferrably non-default gcc configure
option to request it.

	Jakub
Jacob Kroon Nov. 20, 2021, 8:42 a.m. UTC | #3
On 11/20/21 09:32, Jakub Jelinek wrote:
> On Sat, Nov 20, 2021 at 12:24:21AM -0800, Andrew Pinski via Gcc-patches wrote:
>> On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
>> <gcc-patches@gcc.gnu.org> wrote:
>>>
>>> cc1/cc1plus both include a checksum of the object files, archives and
>>> options used during linking. Unless the host binutils has been built
>>> with --enable-deterministic-archives, the archives will have different
>>> checksums each build due to changes in timestamps of the containing
>>> object files, and thus the checksum that is embedded in cc1/cc1plus will
>>> also change.
>>>
>>> Fix this by passing "D" to ar/ranlib when creating the archives.
>>
>> How portable is the D option? That does it work with Mac OS X's ar;
>> what about AIX's ar; what about Solaris's ar, etc?
>> GNU binutils is not the only ar which is supported for the host.
> 
> It isn't portable and even if ar/ranlib do support that option, not all
> users will want it, so forcing it upon them unconditionally is IMO not a
> good idea.  If anything, configure needs to check if those options are
> supported and there needs to be preferrably non-default gcc configure
> option to request it.
> 

Aha, I see. Then perhaps an easier and more portable solution for 
improving the reproducibility would be to make it optional whether or 
not to include the archives when calculating the checksums.

Or perhaps by checksumming the containing object files instead of the 
archives themselves ?

Would something like that be acceptable ?

Jacob
Richard Biener Nov. 22, 2021, 8:51 a.m. UTC | #4
On Sat, Nov 20, 2021 at 9:43 AM Jacob Kroon via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> On 11/20/21 09:32, Jakub Jelinek wrote:
> > On Sat, Nov 20, 2021 at 12:24:21AM -0800, Andrew Pinski via Gcc-patches wrote:
> >> On Sat, Nov 20, 2021 at 12:18 AM Jacob Kroon via Gcc-patches
> >> <gcc-patches@gcc.gnu.org> wrote:
> >>>
> >>> cc1/cc1plus both include a checksum of the object files, archives and
> >>> options used during linking. Unless the host binutils has been built
> >>> with --enable-deterministic-archives, the archives will have different
> >>> checksums each build due to changes in timestamps of the containing
> >>> object files, and thus the checksum that is embedded in cc1/cc1plus will
> >>> also change.
> >>>
> >>> Fix this by passing "D" to ar/ranlib when creating the archives.
> >>
> >> How portable is the D option? That does it work with Mac OS X's ar;
> >> what about AIX's ar; what about Solaris's ar, etc?
> >> GNU binutils is not the only ar which is supported for the host.
> >
> > It isn't portable and even if ar/ranlib do support that option, not all
> > users will want it, so forcing it upon them unconditionally is IMO not a
> > good idea.  If anything, configure needs to check if those options are
> > supported and there needs to be preferrably non-default gcc configure
> > option to request it.
> >
>
> Aha, I see. Then perhaps an easier and more portable solution for
> improving the reproducibility would be to make it optional whether or
> not to include the archives when calculating the checksums.
>
> Or perhaps by checksumming the containing object files instead of the
> archives themselves ?
>
> Would something like that be acceptable ?

At SUSE we are carrying patches to not include the checksum at all
but instead use the linker generated build-id as checksum for PCH.
I suppose when we add the configure option to disable PCH support
we could also elide checksum processing (or as in the patch simply
always use all-zeros).

Attached for reference.

Richard.

>
> Jacob
diff mbox series

Patch

diff --git a/Makefile.in b/Makefile.in
index 860cf8f067b..8c451ed2121 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -416,7 +416,7 @@  MAKEINFOFLAGS = --split-size=5000000
 
 AS = @AS@
 AR = @AR@
-AR_FLAGS = rc
+AR_FLAGS = rcD
 CC = @CC@
 CXX = @CXX@
 DLLTOOL = @DLLTOOL@
@@ -23571,7 +23571,7 @@  configure-libbacktrace:
 	  $$s/$$module_srcdir/configure \
 	  --srcdir=$${topdir}/$$module_srcdir \
 	  $(HOST_CONFIGARGS) --build=${build_alias} --host=${host_alias} \
-	  --target=${target_alias}  \
+	  --target=${target_alias}  RANLIB="ranlib -D" \
 	  || exit 1
 @endif libbacktrace
 
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index 571e9c28e29..828b13d9b54 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -243,7 +243,7 @@  AR = @AR@
 AR_FLAGS = rc
 NM = @NM@
 RANLIB = @RANLIB@
-RANLIB_FLAGS = @ranlib_flags@
+RANLIB_FLAGS = @ranlib_flags@ -D
 
 # Libraries to use on the host.
 HOST_LIBS = @HOST_LIBS@
diff --git a/libbacktrace/configure b/libbacktrace/configure
index a2f33c0f35d..5b18966ea80 100755
--- a/libbacktrace/configure
+++ b/libbacktrace/configure
@@ -6669,7 +6669,7 @@  else
 fi
 
 test -z "$AR" && AR=ar
-test -z "$AR_FLAGS" && AR_FLAGS=cru
+test -z "$AR_FLAGS" && AR_FLAGS=crD
 
 
 
diff --git a/libcody/Makefile.in b/libcody/Makefile.in
index 7eaf8ace8ce..f1e8be352b5 100644
--- a/libcody/Makefile.in
+++ b/libcody/Makefile.in
@@ -126,8 +126,8 @@  LIBCODY.O := buffer.o client.o fatal.o netclient.o netserver.o \
 all:: libcody.a
 
 libcody.a: $(LIBCODY.O)
-	$(AR) -cr $@ $^
-	$(RANLIB) $@
+	$(AR) -crD $@ $^
+	$(RANLIB) -D $@
 
 mostlyclean::
 
diff --git a/libcpp/Makefile.in b/libcpp/Makefile.in
index 34e4206957d..53d220c4e0b 100644
--- a/libcpp/Makefile.in
+++ b/libcpp/Makefile.in
@@ -26,7 +26,7 @@  top_builddir = .
 VPATH = @srcdir@
 INSTALL = @INSTALL@
 AR = @AR@
-ARFLAGS = cru
+ARFLAGS = crD
 ACLOCAL = @ACLOCAL@
 AUTOCONF = @AUTOCONF@
 AUTOHEADER = @AUTOHEADER@
@@ -101,7 +101,7 @@  all: libcpp.a $(USED_CATALOGS)
 libcpp.a: $(libcpp_a_OBJS)
 	-rm -f libcpp.a
 	$(AR) $(ARFLAGS) libcpp.a $(libcpp_a_OBJS)
-	$(RANLIB) libcpp.a
+	$(RANLIB) -D libcpp.a
 
 # Rules to rebuild the configuration
 
diff --git a/libdecnumber/Makefile.in b/libdecnumber/Makefile.in
index 7e086fccafa..3b03064fc4d 100644
--- a/libdecnumber/Makefile.in
+++ b/libdecnumber/Makefile.in
@@ -26,7 +26,7 @@  top_builddir = .
 VPATH = @srcdir@
 INSTALL = @INSTALL@
 AR = @AR@
-ARFLAGS = cru
+ARFLAGS = crD
 ACLOCAL = @ACLOCAL@
 AUTOCONF = @AUTOCONF@
 AUTOHEADER = @AUTOHEADER@
@@ -88,7 +88,7 @@  all: libdecnumber.a
 libdecnumber.a: $(libdecnumber_a_OBJS)
 	-rm -f $@
 	$(AR) $(ARFLAGS) $@ $(libdecnumber_a_OBJS)
-	$(RANLIB) $@
+	$(RANLIB) -D $@
 
 # Rules to rebuild the configuration
 
diff --git a/libiberty/Makefile.in b/libiberty/Makefile.in
index 884cc6c4af2..9f507a54657 100644
--- a/libiberty/Makefile.in
+++ b/libiberty/Makefile.in
@@ -252,19 +252,19 @@  $(TARGETLIB): $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
 	-rm -f $(TARGETLIB) pic/$(TARGETLIB) noasan/$(TARGETLIB)
 	$(AR) $(AR_FLAGS) $(TARGETLIB) \
 	  $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS)
-	$(RANLIB) $(TARGETLIB)
+	$(RANLIB) -D $(TARGETLIB)
 	if [ x"$(PICFLAG)" != x ]; then \
 	  cd pic; \
 	  $(AR) $(AR_FLAGS) $(TARGETLIB) \
 	    $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
-	  $(RANLIB) $(TARGETLIB); \
+	  $(RANLIB) -D $(TARGETLIB); \
 	  cd ..; \
 	else true; fi; \
 	if [ x"$(NOASANFLAG)" != x ]; then \
 	  cd noasan; \
 	  $(AR) $(AR_FLAGS) $(TARGETLIB) \
 	    $(REQUIRED_OFILES) $(EXTRA_OFILES) $(LIBOBJS); \
-	  $(RANLIB) $(TARGETLIB); \
+	  $(RANLIB) -D $(TARGETLIB); \
 	  cd ..; \
 	else true; fi