Check nonlinear iv in vect_can_advance_ivs_p.
Commit Message
vectorizable_nonlinear_induction doesn't always guard
vect_peel_nonlinear_iv_init when it's called by
vect_update_ivs_after_vectorizer which is supposed to be guarded
by vect_can_advance_ivs_p. The patch put part codes from
vectorizable_nonlinear_induction into a new function
vect_can_peel_nonlinear_iv_p, and make vect_can_advcance_ivs_p call it.
Also this patch fix a typo in my initial patch as bellow.
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && induction_type == vect_step_op_mul)
if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,}.
Ok for trunk?
gcc/ChangeLog:
PR tree-optimization/107055
* tree-vect-loop-manip.cc (vect_can_advance_ivs_p): Check for
nonlinear induction variables.
* tree-vect-loop.cc (vect_can_peel_nonlinear_iv_p): New
functions.
(vectorizable_nonlinear_induction): Put part codes into
vect_can_peel_nonlinear_iv_p.
* tree-vectorizer.h (vect_can_peel_nonlinear_iv_p): Declare.
gcc/testsuite/ChangeLog:
* gcc.target/i386/pr107055.c: New test.
---
gcc/testsuite/gcc.target/i386/pr107055.c | 4 ++
gcc/tree-vect-loop-manip.cc | 10 +++
gcc/tree-vect-loop.cc | 82 ++++++++++++++----------
gcc/tree-vectorizer.h | 3 +
4 files changed, 64 insertions(+), 35 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/pr107055.c
Comments
On Thu, Sep 29, 2022 at 7:03 AM liuhongt via Gcc-patches
<gcc-patches@gcc.gnu.org> wrote:
>
> vectorizable_nonlinear_induction doesn't always guard
> vect_peel_nonlinear_iv_init when it's called by
> vect_update_ivs_after_vectorizer which is supposed to be guarded
> by vect_can_advance_ivs_p. The patch put part codes from
> vectorizable_nonlinear_induction into a new function
> vect_can_peel_nonlinear_iv_p, and make vect_can_advcance_ivs_p call it.
>
> Also this patch fix a typo in my initial patch as bellow.
> - if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
> - && induction_type == vect_step_op_mul)
> if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
>
> Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,}.
> Ok for trunk?
OK.
Thanks,
Richard.
> gcc/ChangeLog:
>
> PR tree-optimization/107055
> * tree-vect-loop-manip.cc (vect_can_advance_ivs_p): Check for
> nonlinear induction variables.
> * tree-vect-loop.cc (vect_can_peel_nonlinear_iv_p): New
> functions.
> (vectorizable_nonlinear_induction): Put part codes into
> vect_can_peel_nonlinear_iv_p.
> * tree-vectorizer.h (vect_can_peel_nonlinear_iv_p): Declare.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/i386/pr107055.c: New test.
> ---
> gcc/testsuite/gcc.target/i386/pr107055.c | 4 ++
> gcc/tree-vect-loop-manip.cc | 10 +++
> gcc/tree-vect-loop.cc | 82 ++++++++++++++----------
> gcc/tree-vectorizer.h | 3 +
> 4 files changed, 64 insertions(+), 35 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/pr107055.c
>
> diff --git a/gcc/testsuite/gcc.target/i386/pr107055.c b/gcc/testsuite/gcc.target/i386/pr107055.c
> new file mode 100644
> index 00000000000..63bcb3d742e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/pr107055.c
> @@ -0,0 +1,4 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O1 -fno-tree-dce -fno-vect-cost-model -ftree-vectorize -fprofile-arcs" } */
> +
> +#include "../../gcc.dg/torture/pr24257.c"
> diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
> index 74b221a973c..1d96130c985 100644
> --- a/gcc/tree-vect-loop-manip.cc
> +++ b/gcc/tree-vect-loop-manip.cc
> @@ -1413,6 +1413,7 @@ vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
> for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
> {
> tree evolution_part;
> + enum vect_induction_op_type induction_type;
>
> gphi *phi = gsi.phi ();
> stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
> @@ -1432,6 +1433,15 @@ vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
> continue;
> }
>
> + induction_type = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (phi_info);
> + if (induction_type != vect_step_op_add)
> + {
> + if (!vect_can_peel_nonlinear_iv_p (loop_vinfo, induction_type))
> + return false;
> +
> + continue;
> + }
> +
> /* Analyze the evolution function. */
>
> evolution_part = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index aabdc6f2d81..2536cc3cf49 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -8558,6 +8558,50 @@ vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
> return vec_def;
>
> }
> +
> +/* Return true if vectorizer can peel for nonlinear iv. */
> +bool
> +vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
> + enum vect_induction_op_type induction_type)
> +{
> + tree niters_skip;
> + /* Init_expr will be update by vect_update_ivs_after_vectorizer,
> + if niters is unkown:
> + For shift, when shift mount >= precision, there would be UD.
> + For mult, don't known how to generate
> + init_expr * pow (step, niters) for variable niters.
> + For neg, it should be ok, since niters of vectorized main loop
> + will always be multiple of 2. */
> + if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
> + && induction_type != vect_step_op_neg)
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> + "Peeling for epilogue is not supported"
> + " for nonlinear induction except neg"
> + " when iteration count is unknown.\n");
> + return false;
> + }
> +
> + /* Also doens't support peel for neg when niter is variable.
> + ??? generate something like niter_expr & 1 ? init_expr : -init_expr? */
> + niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
> + if ((niters_skip != NULL_TREE
> + && TREE_CODE (niters_skip) != INTEGER_CST)
> + || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
> + && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> + "Peeling for alignement is not supported"
> + " for nonlinear induction when niters_skip"
> + " is not constant.\n");
> + return false;
> + }
> +
> + return true;
> +}
> +
> /* Function vectorizable_induction
>
> Check if STMT_INFO performs an nonlinear induction computation that can be
> @@ -8628,42 +8672,9 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
> return false;
> }
>
> - /* Init_expr will be update by vect_update_ivs_after_vectorizer,
> - if niters is unkown:
> - For shift, when shift mount >= precision, there would be UD.
> - For mult, don't known how to generate
> - init_expr * pow (step, niters) for variable niters.
> - For neg, it should be ok, since niters of vectorized main loop
> - will always be multiple of 2. */
> - if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
> - && induction_type != vect_step_op_neg)
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "Peeling for epilogue is not supported"
> - " for nonlinear induction except neg"
> - " when iteration count is unknown.\n");
> - return false;
> - }
> -
> - /* Also doens't support peel for neg when niter is variable.
> - ??? generate something like niter_expr & 1 ? init_expr : -init_expr? */
> - niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
> - if ((niters_skip != NULL_TREE
> - && TREE_CODE (niters_skip) != INTEGER_CST)
> - || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
> - && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "Peeling for alignement is not supported"
> - " for nonlinear induction when niters_skip"
> - " is not constant.\n");
> - return false;
> - }
> + if (!vect_can_peel_nonlinear_iv_p (loop_vinfo, induction_type))
> + return false;
>
> - if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
> - && induction_type == vect_step_op_mul)
> if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
> {
> if (dump_enabled_p ())
> @@ -8799,6 +8810,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
>
> gimple_seq stmts = NULL;
>
> + niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
> /* If we are using the loop mask to "peel" for alignment then we need
> to adjust the start value here. */
> if (niters_skip != NULL_TREE)
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index 5e75ed1532b..4870c754499 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -2343,6 +2343,9 @@ extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
> /* Nonlinear induction. */
> extern tree vect_peel_nonlinear_iv_init (gimple_seq*, tree, tree,
> tree, enum vect_induction_op_type);
> +extern bool
> +vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
> + enum vect_induction_op_type induction_type);
>
> /* In tree-vect-slp.cc. */
> extern void vect_slp_init (void);
> --
> 2.18.1
>
new file mode 100644
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fno-tree-dce -fno-vect-cost-model -ftree-vectorize -fprofile-arcs" } */
+
+#include "../../gcc.dg/torture/pr24257.c"
@@ -1413,6 +1413,7 @@ vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
tree evolution_part;
+ enum vect_induction_op_type induction_type;
gphi *phi = gsi.phi ();
stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
@@ -1432,6 +1433,15 @@ vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
continue;
}
+ induction_type = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (phi_info);
+ if (induction_type != vect_step_op_add)
+ {
+ if (!vect_can_peel_nonlinear_iv_p (loop_vinfo, induction_type))
+ return false;
+
+ continue;
+ }
+
/* Analyze the evolution function. */
evolution_part = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
@@ -8558,6 +8558,50 @@ vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
return vec_def;
}
+
+/* Return true if vectorizer can peel for nonlinear iv. */
+bool
+vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
+ enum vect_induction_op_type induction_type)
+{
+ tree niters_skip;
+ /* Init_expr will be update by vect_update_ivs_after_vectorizer,
+ if niters is unkown:
+ For shift, when shift mount >= precision, there would be UD.
+ For mult, don't known how to generate
+ init_expr * pow (step, niters) for variable niters.
+ For neg, it should be ok, since niters of vectorized main loop
+ will always be multiple of 2. */
+ if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && induction_type != vect_step_op_neg)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Peeling for epilogue is not supported"
+ " for nonlinear induction except neg"
+ " when iteration count is unknown.\n");
+ return false;
+ }
+
+ /* Also doens't support peel for neg when niter is variable.
+ ??? generate something like niter_expr & 1 ? init_expr : -init_expr? */
+ niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
+ if ((niters_skip != NULL_TREE
+ && TREE_CODE (niters_skip) != INTEGER_CST)
+ || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
+ && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Peeling for alignement is not supported"
+ " for nonlinear induction when niters_skip"
+ " is not constant.\n");
+ return false;
+ }
+
+ return true;
+}
+
/* Function vectorizable_induction
Check if STMT_INFO performs an nonlinear induction computation that can be
@@ -8628,42 +8672,9 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
return false;
}
- /* Init_expr will be update by vect_update_ivs_after_vectorizer,
- if niters is unkown:
- For shift, when shift mount >= precision, there would be UD.
- For mult, don't known how to generate
- init_expr * pow (step, niters) for variable niters.
- For neg, it should be ok, since niters of vectorized main loop
- will always be multiple of 2. */
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && induction_type != vect_step_op_neg)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Peeling for epilogue is not supported"
- " for nonlinear induction except neg"
- " when iteration count is unknown.\n");
- return false;
- }
-
- /* Also doens't support peel for neg when niter is variable.
- ??? generate something like niter_expr & 1 ? init_expr : -init_expr? */
- niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
- if ((niters_skip != NULL_TREE
- && TREE_CODE (niters_skip) != INTEGER_CST)
- || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
- && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Peeling for alignement is not supported"
- " for nonlinear induction when niters_skip"
- " is not constant.\n");
- return false;
- }
+ if (!vect_can_peel_nonlinear_iv_p (loop_vinfo, induction_type))
+ return false;
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && induction_type == vect_step_op_mul)
if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
{
if (dump_enabled_p ())
@@ -8799,6 +8810,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
gimple_seq stmts = NULL;
+ niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
/* If we are using the loop mask to "peel" for alignment then we need
to adjust the start value here. */
if (niters_skip != NULL_TREE)
@@ -2343,6 +2343,9 @@ extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
/* Nonlinear induction. */
extern tree vect_peel_nonlinear_iv_init (gimple_seq*, tree, tree,
tree, enum vect_induction_op_type);
+extern bool
+vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
+ enum vect_induction_op_type induction_type);
/* In tree-vect-slp.cc. */
extern void vect_slp_init (void);