[1/2] i386: Take the va_arg by-reference decision from the va_list type

  For an MS ABI va_list, ix86_gimplify_va_arg calls
std_gimplify_va_arg_expr, which asks the pass_by_reference hook whether
the argument was passed by reference.  Without cumulative args, the hook
assumes the default ABI, SysV on Linux, so va_arg reads an argument
passed by reference as if it were passed by value.

Add an INDIRECT argument to std_gimplify_va_arg_expr.  For an MS ABI
va_list, ix86_gimplify_va_arg now passes the result of the new
ix86_pass_by_reference_abi with MS_ABI, so the decision follows the
same ABI as the va_list layout.  The SysV va_list path has the opposite
problem on mingw, where the default ABI is MS; it now uses
ix86_pass_by_reference_abi with SYSV_ABI.

Bootstrapped and regression tested on x86_64-pc-linux-gnu.  The new
test also passes on x86_64-w64-mingw32.

gcc/ChangeLog:

	* config/i386/i386.cc (ix86_pass_by_reference_abi): New function,
	split out of ...
	(ix86_pass_by_reference): ... this.
	(ix86_gimplify_va_arg): Use ix86_pass_by_reference_abi with the
	ABI of the va_list type.
	* targhooks.cc (std_gimplify_va_arg_expr): Add an overload taking
	INDIRECT.  Implement the existing one with it.
	* targhooks.h (std_gimplify_va_arg_expr): Declare the overload.

gcc/testsuite/ChangeLog:

	* gcc.target/i386/va-arg-abi-attribute.c: New test.

Signed-off-by: Oleg Tolmatcev <oleg.tolmatcev@gmail.com>
---
 gcc/config/i386/i386.cc                       | 28 +++++--
 gcc/targhooks.cc                              | 31 +++++---
 gcc/targhooks.h                               |  2 +
 .../gcc.target/i386/va-arg-abi-attribute.c    | 73 +++++++++++++++++++
 4 files changed, 117 insertions(+), 17 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/i386/va-arg-abi-attribute.c
  

On Wed, Sep 30, 2026 at 7:47 PM Oleg Tolmatcev <oleg.tolmatcev@gmail.com> wrote:
>
> For an MS ABI va_list, ix86_gimplify_va_arg calls
> std_gimplify_va_arg_expr, which asks the pass_by_reference hook whether
> the argument was passed by reference.  Without cumulative args, the hook
> assumes the default ABI, SysV on Linux, so va_arg reads an argument
> passed by reference as if it were passed by value.
>
> Add an INDIRECT argument to std_gimplify_va_arg_expr.  For an MS ABI
> va_list, ix86_gimplify_va_arg now passes the result of the new
> ix86_pass_by_reference_abi with MS_ABI, so the decision follows the
> same ABI as the va_list layout.  The SysV va_list path has the opposite
> problem on mingw, where the default ABI is MS; it now uses
> ix86_pass_by_reference_abi with SYSV_ABI.
>
> Bootstrapped and regression tested on x86_64-pc-linux-gnu.  The new
> test also passes on x86_64-w64-mingw32.
>
> gcc/ChangeLog:
>
>         * config/i386/i386.cc (ix86_pass_by_reference_abi): New function,
>         split out of ...
>         (ix86_pass_by_reference): ... this.
>         (ix86_gimplify_va_arg): Use ix86_pass_by_reference_abi with the
>         ABI of the va_list type.
>         * targhooks.cc (std_gimplify_va_arg_expr): Add an overload taking
>         INDIRECT.  Implement the existing one with it.
>         * targhooks.h (std_gimplify_va_arg_expr): Declare the overload.
>
> gcc/testsuite/ChangeLog:
>
>         * gcc.target/i386/va-arg-abi-attribute.c: New test.

Please split this patch to target-independant and target part. A
generic target independent patch has different reviewers and it needs
to be approved first.

>
> Signed-off-by: Oleg Tolmatcev <oleg.tolmatcev@gmail.com>
> ---
>  gcc/config/i386/i386.cc                       | 28 +++++--
>  gcc/targhooks.cc                              | 31 +++++---
>  gcc/targhooks.h                               |  2 +
>  .../gcc.target/i386/va-arg-abi-attribute.c    | 73 +++++++++++++++++++
>  4 files changed, 117 insertions(+), 17 deletions(-)
>  create mode 100644 gcc/testsuite/gcc.target/i386/va-arg-abi-attribute.c
>
> diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
> index e9015d2427..98b6b3f9d2 100644
> --- a/gcc/config/i386/i386.cc
> +++ b/gcc/config/i386/i386.cc
> @@ -3655,17 +3655,15 @@ ix86_function_arg (cumulative_args_t cum_v, const function_arg_info &arg)
>     reference.  If nonzero for an argument, a copy of that argument is
>     made in memory and a pointer to the argument is passed instead of
>     the argument itself.  The pointer is passed in whatever way is
> -   appropriate for passing a pointer to that type.  */
> +   appropriate for passing a pointer to that type.  CALL_ABI is the
> +   calling convention the argument is passed under.  */
>
>  static bool
> -ix86_pass_by_reference (cumulative_args_t cum_v, const function_arg_info &arg)
> +ix86_pass_by_reference_abi (enum calling_abi call_abi,
> +                           const function_arg_info &arg)
>  {
> -  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
> -
>    if (TARGET_64BIT)
>      {
> -      enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi;
> -
>        /* See Windows x64 Software Convention.  */
>        if (call_abi == MS_ABI)
>         {
> @@ -3695,6 +3693,16 @@ ix86_pass_by_reference (cumulative_args_t cum_v, const function_arg_info &arg)
>    return false;
>  }
>
> +/* Implement TARGET_PASS_BY_REFERENCE.  */
> +
> +static bool
> +ix86_pass_by_reference (cumulative_args_t cum_v, const function_arg_info &arg)
> +{
> +  CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
> +
> +  return ix86_pass_by_reference_abi (cum ? cum->call_abi : ix86_abi, arg);
> +}
> +
>  /* Return true when TYPE should be 128bit aligned for 32bit argument
>     passing ABI.  XXX: This function is obsolete and is only used for
>     checking psABI compatibility with previous versions of GCC.  */
> @@ -4996,7 +5004,10 @@ ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
>
>    /* Only 64bit target needs something special.  */
>    if (is_va_list_char_pointer (TREE_TYPE (valist)))
> -    return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
> +    return std_gimplify_va_arg_expr (valist, type,
> +                                    ix86_pass_by_reference_abi
> +                                      (MS_ABI, function_arg_info (type, false)),
> +                                    pre_p, post_p);

Uh, oh ... please use indirect_p variable as a temporary here. Please
redeclare it as uninitialized bool:

-  int indirect_p = 0;
+  bool indirect_p;

(all its uses follow initialization in the source code below)

Uros.

>    f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
>    f_fpr = DECL_CHAIN (f_gpr);
> @@ -5010,7 +5021,8 @@ ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
>    ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
>    sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
>
> -  indirect_p = pass_va_arg_by_reference (type);
> +  indirect_p = ix86_pass_by_reference_abi (SYSV_ABI,
> +                                          function_arg_info (type, false));
>    if (indirect_p)
>      type = build_pointer_type (type);
>    size = arg_int_size_in_bytes (type);
> diff --git a/gcc/targhooks.cc b/gcc/targhooks.cc
> index 7123827cfd..1933c01b65 100644
> --- a/gcc/targhooks.cc
> +++ b/gcc/targhooks.cc
> @@ -2624,15 +2624,15 @@ build_va_arg_indirect_ref (tree addr)
>  }
>
>  /* The "standard" implementation of va_arg: read the value from the
> -   current (padded) address and increment by the (padded) size.  */
> +   current (padded) address and increment by the (padded) size.
> +   INDIRECT is true if the argument is passed by reference.  */
>
>  tree
> -std_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
> -                         gimple_seq *post_p)
> +std_gimplify_va_arg_expr (tree valist, tree type, bool indirect,
> +                         gimple_seq *pre_p, gimple_seq *post_p)
>  {
>    tree addr, t, type_size, rounded_size, valist_tmp;
>    unsigned HOST_WIDE_INT align, boundary;
> -  bool indirect;
>
>    /* All of the alignment and movement below is for args-grow-up machines.
>       As of 2004, there are only 3 ARGS_GROW_DOWNWARD targets, and they all
> @@ -2640,7 +2640,6 @@ std_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
>    if (ARGS_GROW_DOWNWARD)
>      gcc_unreachable ();
>
> -  indirect = pass_va_arg_by_reference (type);
>    if (indirect)
>      type = build_pointer_type (type);
>
> @@ -2649,13 +2648,15 @@ std_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
>        && targetm.calls.split_complex_arg (type))
>      {
>        tree real_part, imag_part;
> +      tree part_type = TREE_TYPE (type);
> +      bool part_indirect = pass_va_arg_by_reference (part_type);
>
> -      real_part = std_gimplify_va_arg_expr (valist,
> -                                           TREE_TYPE (type), pre_p, NULL);
> +      real_part = std_gimplify_va_arg_expr (valist, part_type, part_indirect,
> +                                           pre_p, NULL);
>        real_part = get_initialized_tmp_var (real_part, pre_p);
>
> -      imag_part = std_gimplify_va_arg_expr (unshare_expr (valist),
> -                                           TREE_TYPE (type), pre_p, NULL);
> +      imag_part = std_gimplify_va_arg_expr (unshare_expr (valist), part_type,
> +                                           part_indirect, pre_p, NULL);
>        imag_part = get_initialized_tmp_var (imag_part, pre_p);
>
>        return build2 (COMPLEX_EXPR, type, real_part, imag_part);
> @@ -2737,6 +2738,18 @@ std_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
>    return build_va_arg_indirect_ref (addr);
>  }
>
> +/* Likewise, but ask pass_va_arg_by_reference whether TYPE is passed
> +   by reference.  */
> +
> +tree
> +std_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
> +                         gimple_seq *post_p)
> +{
> +  return std_gimplify_va_arg_expr (valist, type,
> +                                  pass_va_arg_by_reference (type),
> +                                  pre_p, post_p);
> +}
> +
>  /* An implementation of TARGET_CAN_USE_DOLOOP_P for targets that do
>     not support nested low-overhead loops.  */
>
> diff --git a/gcc/targhooks.h b/gcc/targhooks.h
> index ffeaeb2a77..9e75e27594 100644
> --- a/gcc/targhooks.h
> +++ b/gcc/targhooks.h
> @@ -292,6 +292,8 @@ extern bool default_member_type_forces_blk (const_tree, machine_mode);
>  extern void default_atomic_assign_expand_fenv (tree *, tree *, tree *);
>  extern tree build_va_arg_indirect_ref (tree);
>  extern tree std_gimplify_va_arg_expr (tree, tree, gimple_seq *, gimple_seq *);
> +extern tree std_gimplify_va_arg_expr (tree, tree, bool, gimple_seq *,
> +                                     gimple_seq *);
>  extern bool can_use_doloop_if_innermost (const widest_int &,
>                                          const widest_int &,
>                                          unsigned int, bool);
> diff --git a/gcc/testsuite/gcc.target/i386/va-arg-abi-attribute.c b/gcc/testsuite/gcc.target/i386/va-arg-abi-attribute.c
> new file mode 100644
> index 0000000000..48aed0c914
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/va-arg-abi-attribute.c
> @@ -0,0 +1,73 @@
> +/* ix86_gimplify_va_arg must decide whether an argument is passed by
> +   reference from the va_list type, not from the ABI of the function
> +   containing the va_arg or from the target default.  */
> +/* { dg-do run { target { lp64 || llp64 } } } */
> +/* { dg-options "-std=gnu99" } */
> +
> +typedef int v4si __attribute__ ((vector_size (16)));
> +struct s3 { char a, b, c; };
> +struct s8 { int a, b; };
> +struct s12 { int a, b, c; };
> +struct s16 { long long a, b; };
> +
> +#define READER(name, abi, list)                                                \
> +static __attribute__ ((abi, noipa)) void                               \
> +name (list ap)                                                         \
> +{                                                                      \
> +  struct s3 x3 = __builtin_va_arg (ap, struct s3);                     \
> +  struct s8 x8 = __builtin_va_arg (ap, struct s8);                     \
> +  struct s12 x12 = __builtin_va_arg (ap, struct s12);                  \
> +  struct s16 x16 = __builtin_va_arg (ap, struct s16);                  \
> +  v4si v = __builtin_va_arg (ap, v4si);                                        \
> +  int i = __builtin_va_arg (ap, int);                                  \
> +  if (x3.a != 1 || x3.b != 2 || x3.c != 3                              \
> +      || x8.a != 4 || x8.b != 5                                                \
> +      || x12.a != 6 || x12.b != 7 || x12.c != 8                                \
> +      || x16.a != 9 || x16.b != 10                                     \
> +      || v[0] != 11 || v[1] != 12 || v[2] != 13 || v[3] != 14          \
> +      || i != 15)                                                      \
> +    __builtin_abort ();                                                        \
> +}
> +
> +READER (sysv_reads_sysv, sysv_abi, __builtin_sysv_va_list)
> +READER (ms_reads_sysv, ms_abi, __builtin_sysv_va_list)
> +READER (sysv_reads_ms, sysv_abi, __builtin_ms_va_list)
> +READER (ms_reads_ms, ms_abi, __builtin_ms_va_list)
> +
> +static __attribute__ ((sysv_abi, noipa)) void
> +sysv_varargs (int n, ...)
> +{
> +  __builtin_sysv_va_list ap;
> +  __builtin_sysv_va_start (ap, n);
> +  sysv_reads_sysv (ap);
> +  __builtin_sysv_va_end (ap);
> +  __builtin_sysv_va_start (ap, n);
> +  ms_reads_sysv (ap);
> +  __builtin_sysv_va_end (ap);
> +}
> +
> +static __attribute__ ((ms_abi, noipa)) void
> +ms_varargs (int n, ...)
> +{
> +  __builtin_ms_va_list ap;
> +  __builtin_ms_va_start (ap, n);
> +  sysv_reads_ms (ap);
> +  __builtin_ms_va_end (ap);
> +  __builtin_ms_va_start (ap, n);
> +  ms_reads_ms (ap);
> +  __builtin_ms_va_end (ap);
> +}
> +
> +int
> +main (void)
> +{
> +  struct s3 x3 = { 1, 2, 3 };
> +  struct s8 x8 = { 4, 5 };
> +  struct s12 x12 = { 6, 7, 8 };
> +  struct s16 x16 = { 9, 10 };
> +  v4si v = { 11, 12, 13, 14 };
> +
> +  sysv_varargs (0, x3, x8, x12, x16, v, 15);
> +  ms_varargs (0, x3, x8, x12, x16, v, 15);
> +  return 0;
> +}
> --
> 2.56.0.windows.1
>