diff mbox series

middle-end: Handle hybrid SLP induction vectorization with early breaks.

Message ID patch-18097-tamar@arm.com
State New
Headers
Series middle-end: Handle hybrid SLP induction vectorization with early breaks. |

Commit Message

Tamar Christina Dec. 19, 2023, 10:12 a.m. UTC 
  Hi All,

While we don't support SLP for early break vectorization, we
can land in the situation where the induction was vectorized
through hybrid SLP.  This means when vectorizing the early
break live operation we need to get the results of the SLP
operation.


Bootstrapped Regtested on aarch64-none-linux-gnu,
x86_64-pc-linux-gnu and no issues.

Ok for master?

Thanks,
Tamar

gcc/ChangeLog:

	* tree-vect-loop.cc (vectorizable_live_operation): Handle SLP.

gcc/testsuite/ChangeLog:

	* gcc.dg/vect/vect-early-break_82.c: New test.

--- inline copy of patch -- 
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c
new file mode 100644
index 0000000000000000000000000000000000000000..f2a6d640f9c0c381cc2af09bd824e272bcfee0b8




--
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c
new file mode 100644
index 0000000000000000000000000000000000000000..f2a6d640f9c0c381cc2af09bd824e272bcfee0b8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_early_break } */
+/* { dg-require-effective-target vect_int } */
+
+/* { dg-additional-options "-Ofast" } */
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */
+
+#include <complex.h>
+
+#define N 1024
+complex double vect_a[N];
+complex double vect_b[N];
+  
+complex double test4(complex double x, complex double t)
+{
+ complex double ret = 0;
+ for (int i = 0; i < N; i++)
+ {
+   vect_a[i] = t + i;
+   if (vect_a[i] == x)
+     return i;
+   vect_a[i] += x * vect_a[i];
+   
+ }
+ return ret;
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 85b81d30c5ab869cb1f7323caabd9fe4648bdc50..0993d184afe068784474ac225768d9f38d76c040 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -10856,8 +10856,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   bitsize = vector_element_bits_tree (vectype);
 
   /* Get the vectorized lhs of STMT and the lane to use (counted in bits).  */
-  tree vec_lhs, bitstart;
-  gimple *vec_stmt;
+  tree vec_lhs, vec_lhs0, bitstart;
+  gimple *vec_stmt, *vec_stmt0;
   if (slp_node)
     {
       gcc_assert (!loop_vinfo
@@ -10868,6 +10868,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
       vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
 
+      /* In case we need to early break vectorize also get the first stmt.  */
+      vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
+      vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0);
+
       /* Get entry to use.  */
       bitstart = bitsize_int (vec_index);
       bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
@@ -10878,6 +10882,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last ();
       vec_lhs = gimple_get_lhs (vec_stmt);
 
+      /* In case we need to early break vectorize also get the first stmt.  */
+      vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+      vec_lhs0 = gimple_get_lhs (vec_stmt0);
+
       /* Get the last lane in the vector.  */
       bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1));
     }
@@ -10917,7 +10925,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 		 so use ->src.  For main exit the merge block is the
 		 destination.  */
 	      basic_block dest = main_exit_edge ? main_e->dest : e->src;
-	      gimple *tmp_vec_stmt = vec_stmt;
 	      tree tmp_vec_lhs = vec_lhs;
 	      tree tmp_bitstart = bitstart;
 
@@ -10928,8 +10935,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	      if (restart_loop
 		  && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
 		{
-		  tmp_vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-		  tmp_vec_lhs = gimple_get_lhs (tmp_vec_stmt);
+		  tmp_vec_lhs = vec_lhs0;
 		  tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
 		}

Comments

Richard Biener Dec. 19, 2023, 12:36 p.m. UTC | #1 
On Tue, 19 Dec 2023, Tamar Christina wrote:

> Hi All,
> 
> While we don't support SLP for early break vectorization, we
> can land in the situation where the induction was vectorized
> through hybrid SLP.  This means when vectorizing the early
> break live operation we need to get the results of the SLP
> operation.
> 
> 
> Bootstrapped Regtested on aarch64-none-linux-gnu,
> x86_64-pc-linux-gnu and no issues.
> 
> Ok for master?

OK.

> Thanks,
> Tamar
> 
> gcc/ChangeLog:
> 
> 	* tree-vect-loop.cc (vectorizable_live_operation): Handle SLP.
> 
> gcc/testsuite/ChangeLog:
> 
> 	* gcc.dg/vect/vect-early-break_82.c: New test.
> 
> --- inline copy of patch -- 
> diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c
> new file mode 100644
> index 0000000000000000000000000000000000000000..f2a6d640f9c0c381cc2af09bd824e272bcfee0b8
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c
> @@ -0,0 +1,27 @@
> +/* { dg-do compile } */
> +/* { dg-require-effective-target vect_early_break } */
> +/* { dg-require-effective-target vect_int } */
> +
> +/* { dg-additional-options "-Ofast" } */
> +
> +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */
> +
> +#include <complex.h>
> +
> +#define N 1024
> +complex double vect_a[N];
> +complex double vect_b[N];
> +  
> +complex double test4(complex double x, complex double t)
> +{
> + complex double ret = 0;
> + for (int i = 0; i < N; i++)
> + {
> +   vect_a[i] = t + i;
> +   if (vect_a[i] == x)
> +     return i;
> +   vect_a[i] += x * vect_a[i];
> +   
> + }
> + return ret;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 85b81d30c5ab869cb1f7323caabd9fe4648bdc50..0993d184afe068784474ac225768d9f38d76c040 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -10856,8 +10856,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>    bitsize = vector_element_bits_tree (vectype);
>  
>    /* Get the vectorized lhs of STMT and the lane to use (counted in bits).  */
> -  tree vec_lhs, bitstart;
> -  gimple *vec_stmt;
> +  tree vec_lhs, vec_lhs0, bitstart;
> +  gimple *vec_stmt, *vec_stmt0;
>    if (slp_node)
>      {
>        gcc_assert (!loop_vinfo
> @@ -10868,6 +10868,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>        vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
>        vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
>  
> +      /* In case we need to early break vectorize also get the first stmt.  */
> +      vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
> +      vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0);
> +
>        /* Get entry to use.  */
>        bitstart = bitsize_int (vec_index);
>        bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
> @@ -10878,6 +10882,10 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>        vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last ();
>        vec_lhs = gimple_get_lhs (vec_stmt);
>  
> +      /* In case we need to early break vectorize also get the first stmt.  */
> +      vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0];
> +      vec_lhs0 = gimple_get_lhs (vec_stmt0);
> +
>        /* Get the last lane in the vector.  */
>        bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1));
>      }
> @@ -10917,7 +10925,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>  		 so use ->src.  For main exit the merge block is the
>  		 destination.  */
>  	      basic_block dest = main_exit_edge ? main_e->dest : e->src;
> -	      gimple *tmp_vec_stmt = vec_stmt;
>  	      tree tmp_vec_lhs = vec_lhs;
>  	      tree tmp_bitstart = bitstart;
>  
> @@ -10928,8 +10935,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>  	      if (restart_loop
>  		  && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
>  		{
> -		  tmp_vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
> -		  tmp_vec_lhs = gimple_get_lhs (tmp_vec_stmt);
> +		  tmp_vec_lhs = vec_lhs0;
>  		  tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
>  		}
>  
> 
> 
> 
> 
>
diff mbox series

Patch

--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_82.c
@@ -0,0 +1,27 @@ 
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_early_break } */
+/* { dg-require-effective-target vect_int } */
+
+/* { dg-additional-options "-Ofast" } */
+
+/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" } } */
+
+#include <complex.h>
+
+#define N 1024
+complex double vect_a[N];
+complex double vect_b[N];
+  
+complex double test4(complex double x, complex double t)
+{
+ complex double ret = 0;
+ for (int i = 0; i < N; i++)
+ {
+   vect_a[i] = t + i;
+   if (vect_a[i] == x)
+     return i;
+   vect_a[i] += x * vect_a[i];
+   
+ }
+ return ret;
+}
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 85b81d30c5ab869cb1f7323caabd9fe4648bdc50..0993d184afe068784474ac225768d9f38d76c040 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -10856,8 +10856,8 @@  vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   bitsize = vector_element_bits_tree (vectype);
 
   /* Get the vectorized lhs of STMT and the lane to use (counted in bits).  */
-  tree vec_lhs, bitstart;
-  gimple *vec_stmt;
+  tree vec_lhs, vec_lhs0, bitstart;
+  gimple *vec_stmt, *vec_stmt0;
   if (slp_node)
     {
       gcc_assert (!loop_vinfo
@@ -10868,6 +10868,10 @@  vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
       vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
 
+      /* In case we need to early break vectorize also get the first stmt.  */
+      vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
+      vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0);
+
       /* Get entry to use.  */
       bitstart = bitsize_int (vec_index);
       bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
@@ -10878,6 +10882,10 @@  vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last ();
       vec_lhs = gimple_get_lhs (vec_stmt);
 
+      /* In case we need to early break vectorize also get the first stmt.  */
+      vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0];
+      vec_lhs0 = gimple_get_lhs (vec_stmt0);
+
       /* Get the last lane in the vector.  */
       bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1));
     }
@@ -10917,7 +10925,6 @@  vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 		 so use ->src.  For main exit the merge block is the
 		 destination.  */
 	      basic_block dest = main_exit_edge ? main_e->dest : e->src;
-	      gimple *tmp_vec_stmt = vec_stmt;
 	      tree tmp_vec_lhs = vec_lhs;
 	      tree tmp_bitstart = bitstart;
 
@@ -10928,8 +10935,7 @@  vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	      if (restart_loop
 		  && STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
 		{
-		  tmp_vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-		  tmp_vec_lhs = gimple_get_lhs (tmp_vec_stmt);
+		  tmp_vec_lhs = vec_lhs0;
 		  tmp_bitstart = build_zero_cst (TREE_TYPE (bitstart));
 		}