GCC: Enable very long gcc command line options - PR111527

Message ID 20240701102244.471359-1-Deepthi.Hemraj@windriver.com
State New
Headers
Series GCC: Enable very long gcc command line options - PR111527 |

Checks

Context Check Description
linaro-tcwg-bot/tcwg_gcc_build--master-aarch64 success Build passed
linaro-tcwg-bot/tcwg_gcc_check--master-aarch64 fail Test failed
linaro-tcwg-bot/tcwg_gcc_build--master-arm success Build passed
linaro-tcwg-bot/tcwg_gcc_check--master-arm fail Test failed

Commit Message

Deepthi.Hemraj@windriver.com July 1, 2024, 10:22 a.m. UTC
  From: Deepthi Hemraj <Deepthi.Hemraj@windriver.com>

For excessively long environment variables i.e >128KB
Store the arguments in a temporary file and collect them back together in collect2.

This commit patches for COLLECT_GCC_OPTIONS issue:
GCC should not limit the length of command line passed to collect2
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111527

The Linux kernel has the following limits on shell commands:
I.  Total number of bytes used to specify arguments must be under 128KB.
II. Each environment variable passed to an executable must be under 128 KiB

In order to circumvent these limitations, many build tools support
response-files, i.e. files that contain the arguments for the executed
command. These are typically passed using @<filepath> syntax.

1: In gcc, the command line arguments passed to subprocesses are
controlled by 'spec-files'. The default spec for cc1plus includes the
spec of CPP (C-preprocessor). Instead of passing the include directory as
path to a response file to the CPP, gcc expanded it and passed them as
a long string. The same problem was in the assembler (as).
https://gcc.gnu.org/onlinedocs/gcc/Spec-Files.html

2: Gcc uses the COLLECT_GCC_OPTIONS environment variable to transfer the
expanded command line to collect2. With many options, this exceeds the limit II.

GCC : Added new Testcase for PR111527

PR111527-2.c : If the command line argument less than 128kb, gcc should use
               COLLECT_GCC_OPTION to communicate and compile fine.
PR111527-3.c : If the command line argument in the range of 128kb to 2mb,
               gcc should copy arguments in a file and use FILE_GCC_OPTIONS
               to communicate and compile fine.
PR111527-4.c : If the command line argument greater than 2mb, gcc should
               fail the compile and report an error. (Expected FAIL)

Signed-off-by: Topi Kuutela <topi.kuutela@nokia.com>
Signed-off-by: Sunil Dora <sunil.dora1988@gmail.com>
---
 gcc/collect2.cc                           | 38 +++++++++++++++++--
 gcc/gcc.cc                                | 36 ++++++++++++++++--
 gcc/testsuite/gcc.dg/longcmd/longcmd.exp  | 16 ++++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-1.c | 46 +++++++++++++++++++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-2.c | 12 ++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-3.c | 13 +++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-4.c | 11 ++++++
 7 files changed, 166 insertions(+), 6 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/longcmd.exp
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-1.c
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-2.c
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-3.c
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-4.c

Signed-off-by: Deepthi Hemraj <Deepthi.Hemraj@windriver.com>
---
 gcc/collect2.cc                           | 38 +++++++++++++++++--
 gcc/gcc.cc                                | 36 ++++++++++++++++--
 gcc/testsuite/gcc.dg/longcmd/longcmd.exp  | 16 ++++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-1.c | 46 +++++++++++++++++++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-2.c | 12 ++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-3.c | 13 +++++++
 gcc/testsuite/gcc.dg/longcmd/pr111527-4.c | 11 ++++++
 7 files changed, 166 insertions(+), 6 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/longcmd.exp
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-1.c
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-2.c
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-3.c
 create mode 100644 gcc/testsuite/gcc.dg/longcmd/pr111527-4.c
  

Patch

diff --git a/gcc/collect2.cc b/gcc/collect2.cc
index 902014a9cc1..c45a13075d5 100644
--- a/gcc/collect2.cc
+++ b/gcc/collect2.cc
@@ -375,6 +375,38 @@  typedef int scanfilter;
    same format, and begins at __DTOR_LIST__.  */
 
 static void scan_prog_file (const char *, scanpass, scanfilter);
+char* getenv_extended(const char* var_name)
+{
+  int file_size;
+  char* buf = NULL;
+
+  char* string = getenv(var_name);
+  if (!string)
+    {
+      char* string = getenv("FILE_GCC_OPTIONS");
+      FILE *fptr;
+      fptr = fopen(string, "r");
+      if (fptr == NULL)
+	return(0);
+      /* Copy contents from temporary file to buffer */
+      if (fseek(fptr, 0, SEEK_END) == -1)
+	return(0);
+      file_size = ftell(fptr);
+      rewind(fptr);
+      buf = (char *)xmalloc(file_size + 1);
+      if (buf == NULL)
+	return(0);
+      if (fread((void *) buf, file_size, 1, fptr) <= 0)
+	{
+	  free(buf);
+	  fatal_error (input_location, "fread failed");
+	  return(0);
+	}
+      buf[file_size] = '\0';
+      return buf;
+    }
+  return string;
+}
 
 
 /* Delete tempfiles and exit function.  */
@@ -1004,7 +1036,7 @@  main (int argc, char **argv)
     /* Now pick up any flags we want early from COLLECT_GCC_OPTIONS
        The LTO options are passed here as are other options that might
        be unsuitable for ld (e.g. -save-temps).  */
-    p = getenv ("COLLECT_GCC_OPTIONS");
+    p = getenv_extended ("COLLECT_GCC_OPTIONS");
     while (p && *p)
       {
 	const char *q = extract_string (&p);
@@ -1200,7 +1232,7 @@  main (int argc, char **argv)
      AIX support needs to know if -shared has been specified before
      parsing commandline arguments.  */
 
-  p = getenv ("COLLECT_GCC_OPTIONS");
+  p = getenv_extended ("COLLECT_GCC_OPTIONS");
   while (p && *p)
     {
       const char *q = extract_string (&p);
@@ -1594,7 +1626,7 @@  main (int argc, char **argv)
       fprintf (stderr, "o_file              = %s\n",
 	       (o_file ? o_file : "not found"));
 
-      ptr = getenv ("COLLECT_GCC_OPTIONS");
+      ptr = getenv_extended ("COLLECT_GCC_OPTIONS");
       if (ptr)
 	fprintf (stderr, "COLLECT_GCC_OPTIONS = %s\n", ptr);
 
diff --git a/gcc/gcc.cc b/gcc/gcc.cc
index d80b604a48d..0e77ac4969a 100644
--- a/gcc/gcc.cc
+++ b/gcc/gcc.cc
@@ -1247,7 +1247,7 @@  static const char *cpp_unique_options =
  %{!E:%{!M:%{!MM:%{!MT:%{!MQ:%{MD|MMD:%{o*:-MQ %*}}}}}}}\
  %{remap} %{%:debug-level-gt(2):-dD}\
  %{!iplugindir*:%{fplugin*:%:find-plugindir()}}\
- %{H} %C %{D*&U*&A*} %{i*} %Z %i\
+ %{H} %C %{D*&U*&A*} %@{i*} %Z %i\
  %{E|M|MM:%W{o*}}\
  %{fdeps-format=*:%{!fdeps-file=*:-fdeps-file=%:join(%{!o:%b.ddi}%{o*:%.ddi%*})}}\
  %{fdeps-format=*:%{!fdeps-target=*:-fdeps-target=%:join(%{!o:%b.o}%{o*:%.o%*})}}";
@@ -1297,7 +1297,7 @@  static const char *asm_options =
 #if HAVE_GNU_AS
 /* If GNU AS is used, then convert -w (no warnings), -I, and -v
    to the assembler equivalents.  */
-"%{v} %{w:-W} %{I*} "
+"%{v} %{w:-W} %@{I*} "
 #endif
 "%(asm_debug_option)"
 ASM_COMPRESS_DEBUG_SPEC
@@ -2957,7 +2957,37 @@  add_to_obstack (char *path, void *data)
 static void
 xputenv (const char *string)
 {
-  env.xput (string);
+  static const size_t MAX_ENV_VAR_LEN = 128*1024;
+  size_t count;
+  FILE *fptr;
+
+  const size_t string_length = strlen(string);
+  if (string_length < MAX_ENV_VAR_LEN)
+    {
+      env.xput (string);
+      return;
+    }
+  /*  For excessively long environment variables i.e >128KB
+      Store the arguments in a temporary file and collect them back together in collect2 */
+  char *temp_file = make_at_file ();
+  fptr = fopen (temp_file, "w");
+  if (fptr == NULL)
+    {
+      fatal_error (input_location, "Cannot open temporary file");
+      return;
+    }
+  /*  Copy contents into temporary file  */
+  count = fwrite(string, sizeof(char), strlen(string), fptr);
+  if (count != string_length)
+    {
+      fatal_error (input_location, "Cannot write into temporary file");
+      return;
+    }
+  char *env_val = (char *) xmalloc(strlen(temp_file) + 18);
+  sprintf(env_val, "FILE_GCC_OPTIONS=%s", temp_file);
+  env.xput (env_val);
+  record_temp_file (temp_file, !save_temps_flag, !save_temps_flag);
+  fclose(fptr);
 }
 
 /* Build a list of search directories from PATHS.
diff --git a/gcc/testsuite/gcc.dg/longcmd/longcmd.exp b/gcc/testsuite/gcc.dg/longcmd/longcmd.exp
new file mode 100644
index 00000000000..0f7d93199ed
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/longcmd/longcmd.exp
@@ -0,0 +1,16 @@ 
+# GCC testsuite that uses the `dg.exp' driver.
+# Load support procs.
+load_lib gcc-dg.exp
+
+# If a testcase doesn't have special options, use these.
+global DEFAULT_CFLAGS
+if ![info exists DEFAULT_CFLAGS] then {
+    set DEFAULT_CFLAGS " -ansi -pedantic-errors"
+}
+
+dg-init
+
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/*.\[cS\]]] \
+    "" $DEFAULT_CFLAGS
+
+dg-finish
diff --git a/gcc/testsuite/gcc.dg/longcmd/pr111527-1.c b/gcc/testsuite/gcc.dg/longcmd/pr111527-1.c
new file mode 100644
index 00000000000..3137236ab04
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/longcmd/pr111527-1.c
@@ -0,0 +1,46 @@ 
+/* { dg-do run } */
+/* { dg-options "-O2" } */
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+#define ARGV_LIMIT_SIZE              128 * 1024
+#define SYSTEM_LIMIT_SIZE            2 * 1024 * 1024
+#define STR_TO_WRITE                 "-DTEST "
+
+void create_large_response_file ()
+{
+  FILE *fp1, *fp2;
+  char buffer[1024];
+
+  strcpy (buffer, STR_TO_WRITE);
+
+  fp1 = fopen ("options_128kb_to_2mb.txt", "wb");
+  if (fp1 == NULL)
+    {
+      abort ();
+    }
+  while (ftell (fp1) < (ARGV_LIMIT_SIZE + 10))
+    {
+      fwrite (buffer, strlen (buffer), 1, fp1);
+    }
+  fclose (fp1);
+
+  fp2 = fopen ("options_greater_then_2mb.txt", "wb");
+  if (fp2 == NULL)
+    {
+      abort ();
+    }
+  while (ftell (fp2) < (SYSTEM_LIMIT_SIZE +10))
+    {
+      fwrite (buffer, strlen (buffer), 1, fp2);
+    }
+  fclose (fp2);
+}
+
+int main()
+{
+  create_large_response_file ();
+}
diff --git a/gcc/testsuite/gcc.dg/longcmd/pr111527-2.c b/gcc/testsuite/gcc.dg/longcmd/pr111527-2.c
new file mode 100644
index 00000000000..0036bf20672
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/longcmd/pr111527-2.c
@@ -0,0 +1,12 @@ 
+/* { dg-do run } */
+
+/* To ensure gcc works fine and uses COLLECT_GCC_OPTIONS for communication */
+
+#include <stdio.h>
+
+int main()
+{
+  printf("Hello World\n");
+  return 0;
+}
+/* { dg-final { output-exists { target *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/longcmd/pr111527-3.c b/gcc/testsuite/gcc.dg/longcmd/pr111527-3.c
new file mode 100644
index 00000000000..6556ceac064
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/longcmd/pr111527-3.c
@@ -0,0 +1,13 @@ 
+/* { dg-do run } */
+/* { dg-options "@options_128kb_to_2mb.txt" } */
+
+/* Large response file option created in pr111527-1.c  file */
+
+#include <stdio.h>
+
+int main()
+{
+  printf("Hello world\n");
+  return 0;
+}
+/* { dg-final { output-exists { target *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/longcmd/pr111527-4.c b/gcc/testsuite/gcc.dg/longcmd/pr111527-4.c
new file mode 100644
index 00000000000..5386103fcf0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/longcmd/pr111527-4.c
@@ -0,0 +1,11 @@ 
+/* { dg-do compile } { xfail *-*-* } */
+/* { dg-options "@options_greater_then_2mb.txt" } */
+/* { dg-excess-errors "warnings about argument list too long" } */
+
+/* Large response file option created in pr111527-1.c file */
+
+#include <stdio.h>
+
+int main()
+{
+}