Count miscellaneous files built on host for testing as tests

Message ID Pine.LNX.4.64.1403070211280.23661@digraph.polyomino.org.uk
State Committed
Headers show

Commit Message

Joseph Myers March 7, 2014, 2:12 a.m. UTC
In <https://sourceware.org/ml/libc-alpha/2014-01/msg00198.html> I
raised the question of counting miscellaneous dependencies of tests,
built on the host rather than the build system, as tests, so that when
test failures don't stop "make check" neither do those other builds on
the host, so that a flaky host doesn't stop "make check" from
producing a complete summary of test results.  Brooks supported that
idea in <https://sourceware.org/ml/libc-alpha/2014-02/msg00301.html>.

This patch implements that change for all the examples I could find:
one message catalog in catgets/, locales in localedata/ and timezone
files in timezone/.

Tested x86_64.

2014-03-07  Joseph Myers  <joseph@codesourcery.com>

	* catgets/Makefile (tests-special): Add $(objpfx)sample.SJIS.cat.
	($(objpfx)sample.SJIS.cat): Use $(evaluate-test).
	* timezone/Makefile (testdata): Move definition above include of
	Rules.
	(test-zones): New variable.
	(tests-special): Add zone files.
	(build-testdata): Use $(evaluate-test).

localedata/ChangeLog:
2014-03-07  Joseph Myers  <joseph@codesourcery.com>

	* Makefile (LOCALES): Move definition above include of Rules.
	(LOCALE_SRCS): Likewise.
	(CHARMAPS): Likewise.
	(CTYPE_FILES): Likewise.
	(tests-special): Add locale files.
	($(addprefix $(objpfx),$(CTYPE_FILES))): Use $(evaluate-test).

Comments

Carlos O'Donell March 7, 2014, 2:53 a.m. UTC | #1
On 03/06/2014 09:12 PM, Joseph S. Myers wrote:
> In <https://sourceware.org/ml/libc-alpha/2014-01/msg00198.html> I
> raised the question of counting miscellaneous dependencies of tests,
> built on the host rather than the build system, as tests, so that when
> test failures don't stop "make check" neither do those other builds on
> the host, so that a flaky host doesn't stop "make check" from
> producing a complete summary of test results.  Brooks supported that
> idea in <https://sourceware.org/ml/libc-alpha/2014-02/msg00301.html>.
> 
> This patch implements that change for all the examples I could find:
> one message catalog in catgets/, locales in localedata/ and timezone
> files in timezone/.

This looks good to me.

I did not go through to verify we caught all possible instances, just
that the changes here look correct.
 
> Tested x86_64.
> 
> 2014-03-07  Joseph Myers  <joseph@codesourcery.com>
> 
> 	* catgets/Makefile (tests-special): Add $(objpfx)sample.SJIS.cat.
> 	($(objpfx)sample.SJIS.cat): Use $(evaluate-test).
> 	* timezone/Makefile (testdata): Move definition above include of
> 	Rules.
> 	(test-zones): New variable.
> 	(tests-special): Add zone files.
> 	(build-testdata): Use $(evaluate-test).
> 
> localedata/ChangeLog:
> 2014-03-07  Joseph Myers  <joseph@codesourcery.com>
> 
> 	* Makefile (LOCALES): Move definition above include of Rules.
> 	(LOCALE_SRCS): Likewise.
> 	(CHARMAPS): Likewise.
> 	(CTYPE_FILES): Likewise.
> 	(tests-special): Add locale files.
> 	($(addprefix $(objpfx),$(CTYPE_FILES))): Use $(evaluate-test).
> 
> diff --git a/catgets/Makefile b/catgets/Makefile
> index 02ca263..ed06508 100644
> --- a/catgets/Makefile
> +++ b/catgets/Makefile
> @@ -33,7 +33,7 @@ test-srcs = test-gencat
>  
>  ifeq ($(run-built-tests),yes)
>  tests-special += $(objpfx)de/libc.cat $(objpfx)test1.cat $(objpfx)test2.cat \
> -		 $(objpfx)test-gencat.out
> +		 $(objpfx)sample.SJIS.cat $(objpfx)test-gencat.out

OK.

>  endif
>  
>  gencat-modules	= xmalloc
> @@ -88,5 +88,6 @@ $(objpfx)test-gencat.out: test-gencat.sh $(objpfx)test-gencat \
>  
>  $(objpfx)sample.SJIS.cat: sample.SJIS $(objpfx)gencat
>  	GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
> -	$(built-program-cmd) -H $(objpfx)test-gencat.h < $(word 1,$^) > $@
> +	$(built-program-cmd) -H $(objpfx)test-gencat.h < $(word 1,$^) > $@; \
> +	$(evaluate-test)

OK.

>  endif
> diff --git a/localedata/Makefile b/localedata/Makefile
> index 5f01b61..4cf88c8 100644
> --- a/localedata/Makefile
> +++ b/localedata/Makefile
> @@ -97,6 +97,21 @@ tests-special += $(objpfx)sort-test.out $(objpfx)tst-fmon.out \
>  		 $(objpfx)tst-ctype.out $(objpfx)tst-wctype.out \
>  		 $(objpfx)tst-langinfo.out $(objpfx)tst-numeric.out
>  
> +ifeq ($(run-built-tests),yes)
> +# We have to generate locales
> +LOCALES := de_DE.ISO-8859-1 de_DE.UTF-8 en_US.ANSI_X3.4-1968 \
> +	   en_US.ISO-8859-1 en_US.UTF-8 ja_JP.EUC-JP da_DK.ISO-8859-1 \
> +	   hr_HR.ISO-8859-2 sv_SE.ISO-8859-1 ja_JP.SJIS fr_FR.ISO-8859-1 \
> +	   nb_NO.ISO-8859-1 nn_NO.ISO-8859-1 tr_TR.UTF-8 cs_CZ.UTF-8 \
> +	   zh_TW.EUC-TW fa_IR.UTF-8 fr_FR.UTF-8 ja_JP.UTF-8 si_LK.UTF-8 \
> +	   tr_TR.ISO-8859-9
> +LOCALE_SRCS := $(shell echo "$(LOCALES)"|sed 's/\([^ .]*\)[^ ]*/\1/g')
> +CHARMAPS := $(shell echo "$(LOCALES)" | \
> +		    sed -e 's/[^ .]*[.]\([^ ]*\)/\1/g' -e s/SJIS/SHIFT_JIS/g)
> +CTYPE_FILES = $(addsuffix /LC_CTYPE,$(LOCALES))
> +tests-special += $(addprefix $(objpfx),$(CTYPE_FILES))
> +endif
> +
>  include ../Rules
>  
>  # Install the charmap files in gzipped format.
> @@ -119,18 +134,6 @@ CFLAGS-tst-trans.c = -Wno-format
>  
>  
>  ifeq ($(run-built-tests),yes)
> -# We have to generate locales
> -LOCALES := de_DE.ISO-8859-1 de_DE.UTF-8 en_US.ANSI_X3.4-1968 \
> -	   en_US.ISO-8859-1 en_US.UTF-8 ja_JP.EUC-JP da_DK.ISO-8859-1 \
> -	   hr_HR.ISO-8859-2 sv_SE.ISO-8859-1 ja_JP.SJIS fr_FR.ISO-8859-1 \
> -	   nb_NO.ISO-8859-1 nn_NO.ISO-8859-1 tr_TR.UTF-8 cs_CZ.UTF-8 \
> -	   zh_TW.EUC-TW fa_IR.UTF-8 fr_FR.UTF-8 ja_JP.UTF-8 si_LK.UTF-8 \
> -	   tr_TR.ISO-8859-9
> -LOCALE_SRCS := $(shell echo "$(LOCALES)"|sed 's/\([^ .]*\)[^ ]*/\1/g')
> -CHARMAPS := $(shell echo "$(LOCALES)" | \
> -		    sed -e 's/[^ .]*[.]\([^ ]*\)/\1/g' -e s/SJIS/SHIFT_JIS/g)
> -CTYPE_FILES = $(addsuffix /LC_CTYPE,$(LOCALES))
> -

OK.

>  generated-dirs += $(LOCALES)
>  
>  # Dependency for the locale files.  We actually make it depend only on
> @@ -138,7 +141,8 @@ generated-dirs += $(LOCALES)
>  $(addprefix $(objpfx),$(CTYPE_FILES)): %: \
>    gen-locale.sh $(common-objpfx)locale/localedef Makefile \
>    $(addprefix charmaps/,$(CHARMAPS)) $(addprefix locales/,$(LOCALE_SRCS))
> -	@$(SHELL) gen-locale.sh $(common-objpfx) '$(built-program-cmd)' $@
> +	@$(SHELL) gen-locale.sh $(common-objpfx) '$(built-program-cmd)' $@; \
> +	$(evaluate-test)

OK.

>  
>  $(addsuffix .out,$(addprefix $(objpfx),$(tests))): %: \
>    $(addprefix $(objpfx),$(CTYPE_FILES))
> diff --git a/timezone/Makefile b/timezone/Makefile
> index e308cca..998cd14 100644
> --- a/timezone/Makefile
> +++ b/timezone/Makefile
> @@ -45,6 +45,14 @@ CPPFLAGS-zic = -DNOT_IN_libc
>  install-bin-script = tzselect
>  generated += tzselect
>  
> +testdata = $(objpfx)testdata
> +
> +# List zones generated by separate commands running zic on the host.
> +# Each such zic run counts as a separate test.
> +test-zones := America/New_York Etc/UTC UTC Europe/Berlin \
> +	      Australia/Melbourne America/Sao_Paulo Asia/Tokyo
> +tests-special += $(addprefix $(testdata)/, $(test-zones))
> +
>  include ../Rules
>  
>  
> @@ -71,10 +79,10 @@ CFLAGS-scheck.c = -Wno-strict-prototypes -DNOID -DHAVE_GETTEXT
>  # We have to make sure the data for testing the tz functions is available.
>  # Don't add leapseconds here since test-tz made checks that work only without
>  # leapseconds.
> -testdata = $(objpfx)testdata

OK.

>  define build-testdata
>  GCONV_PATH=${common-objpfx}iconvdata LANGUAGE=C LC_ALL=C \
> -  $(built-program-cmd) -d $(testdata) -y ./yearistype $<
> +  $(built-program-cmd) -d $(testdata) -y ./yearistype $<; \
> +$(evaluate-test)

OK.

>  endef
>  
>  $(objpfx)test-tz.out: $(addprefix $(testdata)/, America/New_York Etc/UTC UTC)

Cheers,
Carlos.

Patch

diff --git a/catgets/Makefile b/catgets/Makefile
index 02ca263..ed06508 100644
--- a/catgets/Makefile
+++ b/catgets/Makefile
@@ -33,7 +33,7 @@  test-srcs = test-gencat
 
 ifeq ($(run-built-tests),yes)
 tests-special += $(objpfx)de/libc.cat $(objpfx)test1.cat $(objpfx)test2.cat \
-		 $(objpfx)test-gencat.out
+		 $(objpfx)sample.SJIS.cat $(objpfx)test-gencat.out
 endif
 
 gencat-modules	= xmalloc
@@ -88,5 +88,6 @@  $(objpfx)test-gencat.out: test-gencat.sh $(objpfx)test-gencat \
 
 $(objpfx)sample.SJIS.cat: sample.SJIS $(objpfx)gencat
 	GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
-	$(built-program-cmd) -H $(objpfx)test-gencat.h < $(word 1,$^) > $@
+	$(built-program-cmd) -H $(objpfx)test-gencat.h < $(word 1,$^) > $@; \
+	$(evaluate-test)
 endif
diff --git a/localedata/Makefile b/localedata/Makefile
index 5f01b61..4cf88c8 100644
--- a/localedata/Makefile
+++ b/localedata/Makefile
@@ -97,6 +97,21 @@  tests-special += $(objpfx)sort-test.out $(objpfx)tst-fmon.out \
 		 $(objpfx)tst-ctype.out $(objpfx)tst-wctype.out \
 		 $(objpfx)tst-langinfo.out $(objpfx)tst-numeric.out
 
+ifeq ($(run-built-tests),yes)
+# We have to generate locales
+LOCALES := de_DE.ISO-8859-1 de_DE.UTF-8 en_US.ANSI_X3.4-1968 \
+	   en_US.ISO-8859-1 en_US.UTF-8 ja_JP.EUC-JP da_DK.ISO-8859-1 \
+	   hr_HR.ISO-8859-2 sv_SE.ISO-8859-1 ja_JP.SJIS fr_FR.ISO-8859-1 \
+	   nb_NO.ISO-8859-1 nn_NO.ISO-8859-1 tr_TR.UTF-8 cs_CZ.UTF-8 \
+	   zh_TW.EUC-TW fa_IR.UTF-8 fr_FR.UTF-8 ja_JP.UTF-8 si_LK.UTF-8 \
+	   tr_TR.ISO-8859-9
+LOCALE_SRCS := $(shell echo "$(LOCALES)"|sed 's/\([^ .]*\)[^ ]*/\1/g')
+CHARMAPS := $(shell echo "$(LOCALES)" | \
+		    sed -e 's/[^ .]*[.]\([^ ]*\)/\1/g' -e s/SJIS/SHIFT_JIS/g)
+CTYPE_FILES = $(addsuffix /LC_CTYPE,$(LOCALES))
+tests-special += $(addprefix $(objpfx),$(CTYPE_FILES))
+endif
+
 include ../Rules
 
 # Install the charmap files in gzipped format.
@@ -119,18 +134,6 @@  CFLAGS-tst-trans.c = -Wno-format
 
 
 ifeq ($(run-built-tests),yes)
-# We have to generate locales
-LOCALES := de_DE.ISO-8859-1 de_DE.UTF-8 en_US.ANSI_X3.4-1968 \
-	   en_US.ISO-8859-1 en_US.UTF-8 ja_JP.EUC-JP da_DK.ISO-8859-1 \
-	   hr_HR.ISO-8859-2 sv_SE.ISO-8859-1 ja_JP.SJIS fr_FR.ISO-8859-1 \
-	   nb_NO.ISO-8859-1 nn_NO.ISO-8859-1 tr_TR.UTF-8 cs_CZ.UTF-8 \
-	   zh_TW.EUC-TW fa_IR.UTF-8 fr_FR.UTF-8 ja_JP.UTF-8 si_LK.UTF-8 \
-	   tr_TR.ISO-8859-9
-LOCALE_SRCS := $(shell echo "$(LOCALES)"|sed 's/\([^ .]*\)[^ ]*/\1/g')
-CHARMAPS := $(shell echo "$(LOCALES)" | \
-		    sed -e 's/[^ .]*[.]\([^ ]*\)/\1/g' -e s/SJIS/SHIFT_JIS/g)
-CTYPE_FILES = $(addsuffix /LC_CTYPE,$(LOCALES))
-
 generated-dirs += $(LOCALES)
 
 # Dependency for the locale files.  We actually make it depend only on
@@ -138,7 +141,8 @@  generated-dirs += $(LOCALES)
 $(addprefix $(objpfx),$(CTYPE_FILES)): %: \
   gen-locale.sh $(common-objpfx)locale/localedef Makefile \
   $(addprefix charmaps/,$(CHARMAPS)) $(addprefix locales/,$(LOCALE_SRCS))
-	@$(SHELL) gen-locale.sh $(common-objpfx) '$(built-program-cmd)' $@
+	@$(SHELL) gen-locale.sh $(common-objpfx) '$(built-program-cmd)' $@; \
+	$(evaluate-test)
 
 $(addsuffix .out,$(addprefix $(objpfx),$(tests))): %: \
   $(addprefix $(objpfx),$(CTYPE_FILES))
diff --git a/timezone/Makefile b/timezone/Makefile
index e308cca..998cd14 100644
--- a/timezone/Makefile
+++ b/timezone/Makefile
@@ -45,6 +45,14 @@  CPPFLAGS-zic = -DNOT_IN_libc
 install-bin-script = tzselect
 generated += tzselect
 
+testdata = $(objpfx)testdata
+
+# List zones generated by separate commands running zic on the host.
+# Each such zic run counts as a separate test.
+test-zones := America/New_York Etc/UTC UTC Europe/Berlin \
+	      Australia/Melbourne America/Sao_Paulo Asia/Tokyo
+tests-special += $(addprefix $(testdata)/, $(test-zones))
+
 include ../Rules
 
 
@@ -71,10 +79,10 @@  CFLAGS-scheck.c = -Wno-strict-prototypes -DNOID -DHAVE_GETTEXT
 # We have to make sure the data for testing the tz functions is available.
 # Don't add leapseconds here since test-tz made checks that work only without
 # leapseconds.
-testdata = $(objpfx)testdata
 define build-testdata
 GCONV_PATH=${common-objpfx}iconvdata LANGUAGE=C LC_ALL=C \
-  $(built-program-cmd) -d $(testdata) -y ./yearistype $<
+  $(built-program-cmd) -d $(testdata) -y ./yearistype $<; \
+$(evaluate-test)
 endef
 
 $(objpfx)test-tz.out: $(addprefix $(testdata)/, America/New_York Etc/UTC UTC)