Refactor try_vectorize_loop_1
Commit Message
This refactors epilogue loop handling in try_vectorize_loop_1 to not
suggest we're analyzing those there by splitting out the transform
phase which then can handle the epilogues.
Bootstrapped and tested on x86_64-unknown-linux-gnu, pushed.
2021-10-27 Richard Biener <rguenther@suse.de>
* tree-vectorizer.c (vect_transform_loops): New function,
split out from ...
(try_vectorize_loop_1): ... here. Simplify as epilogues
are now fully handled in the split part.
---
gcc/tree-vectorizer.c | 105 ++++++++++++++++++++----------------------
1 file changed, 50 insertions(+), 55 deletions(-)
@@ -979,6 +979,50 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
free (bbs);
}
+/* Generate vectorized code for LOOP and its epilogues. */
+
+static void
+vect_transform_loops (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+ loop_p loop, gimple *loop_vectorized_call)
+{
+ loop_vec_info loop_vinfo = loop_vec_info_for_loop (loop);
+
+ if (loop_vectorized_call)
+ set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
+
+ unsigned HOST_WIDE_INT bytes;
+ if (dump_enabled_p ())
+ {
+ if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes))
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ "loop vectorized using %wu byte vectors\n", bytes);
+ else
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ "loop vectorized using variable length vectors\n");
+ }
+
+ loop_p new_loop = vect_transform_loop (loop_vinfo,
+ loop_vectorized_call);
+ /* Now that the loop has been vectorized, allow it to be unrolled
+ etc. */
+ loop->force_vectorize = false;
+
+ if (loop->simduid)
+ {
+ simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
+ if (!simduid_to_vf_htab)
+ simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
+ simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
+ simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
+ *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
+ = simduid_to_vf_data;
+ }
+
+ /* Epilogue of vectorized loop must be vectorized too. */
+ if (new_loop)
+ vect_transform_loops (simduid_to_vf_htab, new_loop, NULL);
+}
+
/* Try to vectorize LOOP. */
static unsigned
@@ -999,17 +1043,9 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
LOCATION_FILE (vect_location.get_location_t ()),
LOCATION_LINE (vect_location.get_location_t ()));
- opt_loop_vec_info loop_vinfo = opt_loop_vec_info::success (NULL);
- /* In the case of epilogue vectorization the loop already has its
- loop_vec_info set, we do not require to analyze the loop in this case. */
- if (loop_vec_info vinfo = loop_vec_info_for_loop (loop))
- loop_vinfo = opt_loop_vec_info::success (vinfo);
- else
- {
- /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */
- loop_vinfo = vect_analyze_loop (loop, &shared);
- loop->aux = loop_vinfo;
- }
+ /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */
+ opt_loop_vec_info loop_vinfo = vect_analyze_loop (loop, &shared);
+ loop->aux = loop_vinfo;
if (!loop_vinfo)
if (dump_enabled_p ())
@@ -1083,8 +1119,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
return ret;
}
- /* Only count the original scalar loops. */
- if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo) && !dbg_cnt (vect_loop))
+ if (!dbg_cnt (vect_loop))
{
/* Free existing information if loop is analyzed with some
assumptions. */
@@ -1093,62 +1128,22 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
return ret;
}
- if (loop_vectorized_call)
- set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
-
- unsigned HOST_WIDE_INT bytes;
- if (dump_enabled_p ())
- {
- if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes))
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
- "loop vectorized using %wu byte vectors\n", bytes);
- else
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
- "loop vectorized using variable length vectors\n");
- }
-
- loop_p new_loop = vect_transform_loop (loop_vinfo,
- loop_vectorized_call);
(*num_vectorized_loops)++;
- /* Now that the loop has been vectorized, allow it to be unrolled
- etc. */
- loop->force_vectorize = false;
-
- if (loop->simduid)
- {
- simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
- if (!simduid_to_vf_htab)
- simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
- simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
- simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
- *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
- = simduid_to_vf_data;
- }
+ /* Transform LOOP and its epilogues. */
+ vect_transform_loops (simduid_to_vf_htab, loop, loop_vectorized_call);
if (loop_vectorized_call)
{
fold_loop_internal_call (loop_vectorized_call, boolean_true_node);
- loop_vectorized_call = NULL;
ret |= TODO_cleanup_cfg;
}
if (loop_dist_alias_call)
{
tree value = gimple_call_arg (loop_dist_alias_call, 1);
fold_loop_internal_call (loop_dist_alias_call, value);
- loop_dist_alias_call = NULL;
ret |= TODO_cleanup_cfg;
}
- /* Epilogue of vectorized loop must be vectorized too. */
- if (new_loop)
- {
- /* Don't include vectorized epilogues in the "vectorized loops" count.
- */
- unsigned dont_count = *num_vectorized_loops;
- ret |= try_vectorize_loop_1 (simduid_to_vf_htab, &dont_count,
- new_loop, NULL, NULL);
- }
-
return ret;
}