From patchwork Wed Oct 27 09:30:03 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Richard Biener X-Patchwork-Id: 46700 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id 796C93858403 for ; Wed, 27 Oct 2021 09:30:34 +0000 (GMT) DKIM-Filter: OpenDKIM Filter v2.11.0 sourceware.org 796C93858403 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gcc.gnu.org; s=default; t=1635327034; bh=ANlu65jJpWDnggnnn/D8z6yRgx3xzIk+W7csU0W+Q3g=; h=Date:To:Subject:List-Id:List-Unsubscribe:List-Archive:List-Post: List-Help:List-Subscribe:From:Reply-To:From; b=gzA9yfXuumSiGm6c83OxZ2sa58gEm/iddFBvLKRTwxPTFoyqyA78z2QXjAaqbwTr+ M9SYVKUUjIBgJrVquUlWWbFIQzOZG4dGPp4DC3+HCcrSgRmjndEV5QQmW5Cnwpkp0t jeJL1tYtCqT9MfiQcsFIHPsrGFS2Yb3fFYAV6XSA= X-Original-To: gcc-patches@gcc.gnu.org Delivered-To: gcc-patches@gcc.gnu.org Received: from smtp-out1.suse.de (smtp-out1.suse.de [195.135.220.28]) by sourceware.org (Postfix) with ESMTPS id 172703858401 for ; Wed, 27 Oct 2021 09:30:05 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 172703858401 Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by smtp-out1.suse.de (Postfix) with ESMTPS id BE9CA2195A for ; Wed, 27 Oct 2021 09:30:03 +0000 (UTC) Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by imap2.suse-dmz.suse.de (Postfix) with ESMTPS id AAD0413B69 for ; Wed, 27 Oct 2021 09:30:03 +0000 (UTC) Received: from dovecot-director2.suse.de ([192.168.254.65]) by imap2.suse-dmz.suse.de with ESMTPSA id ddyLKBsceWEmLAAAMHmgww (envelope-from ) for ; Wed, 27 Oct 2021 09:30:03 +0000 Date: Wed, 27 Oct 2021 11:30:03 +0200 (CEST) To: gcc-patches@gcc.gnu.org Subject: [PATCH] Refactor try_vectorize_loop_1 Message-ID: <3nns7662-41n4-6648-440-r870891sssp3@fhfr.qr> MIME-Version: 1.0 X-Spam-Status: No, score=-11.6 required=5.0 tests=BAYES_00, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, GIT_PATCH_0, KAM_NUMSUBJECT, SPF_HELO_NONE, SPF_PASS, TXREP autolearn=ham autolearn_force=no version=3.4.4 X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on server2.sourceware.org X-BeenThere: gcc-patches@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-patches mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-Patchwork-Original-From: Richard Biener via Gcc-patches From: Richard Biener Reply-To: Richard Biener Errors-To: gcc-patches-bounces+patchwork=sourceware.org@gcc.gnu.org Sender: "Gcc-patches" This refactors epilogue loop handling in try_vectorize_loop_1 to not suggest we're analyzing those there by splitting out the transform phase which then can handle the epilogues. Bootstrapped and tested on x86_64-unknown-linux-gnu, pushed. 2021-10-27 Richard Biener * tree-vectorizer.c (vect_transform_loops): New function, split out from ... (try_vectorize_loop_1): ... here. Simplify as epilogues are now fully handled in the split part. --- gcc/tree-vectorizer.c | 105 ++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 4712dc6e7f9..89fa883fbb9 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -979,6 +979,50 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) free (bbs); } +/* Generate vectorized code for LOOP and its epilogues. */ + +static void +vect_transform_loops (hash_table *&simduid_to_vf_htab, + loop_p loop, gimple *loop_vectorized_call) +{ + loop_vec_info loop_vinfo = loop_vec_info_for_loop (loop); + + if (loop_vectorized_call) + set_uid_loop_bbs (loop_vinfo, loop_vectorized_call); + + unsigned HOST_WIDE_INT bytes; + if (dump_enabled_p ()) + { + if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes)) + dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, + "loop vectorized using %wu byte vectors\n", bytes); + else + dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, + "loop vectorized using variable length vectors\n"); + } + + loop_p new_loop = vect_transform_loop (loop_vinfo, + loop_vectorized_call); + /* Now that the loop has been vectorized, allow it to be unrolled + etc. */ + loop->force_vectorize = false; + + if (loop->simduid) + { + simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf); + if (!simduid_to_vf_htab) + simduid_to_vf_htab = new hash_table (15); + simduid_to_vf_data->simduid = DECL_UID (loop->simduid); + simduid_to_vf_data->vf = loop_vinfo->vectorization_factor; + *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT) + = simduid_to_vf_data; + } + + /* Epilogue of vectorized loop must be vectorized too. */ + if (new_loop) + vect_transform_loops (simduid_to_vf_htab, new_loop, NULL); +} + /* Try to vectorize LOOP. */ static unsigned @@ -999,17 +1043,9 @@ try_vectorize_loop_1 (hash_table *&simduid_to_vf_htab, LOCATION_FILE (vect_location.get_location_t ()), LOCATION_LINE (vect_location.get_location_t ())); - opt_loop_vec_info loop_vinfo = opt_loop_vec_info::success (NULL); - /* In the case of epilogue vectorization the loop already has its - loop_vec_info set, we do not require to analyze the loop in this case. */ - if (loop_vec_info vinfo = loop_vec_info_for_loop (loop)) - loop_vinfo = opt_loop_vec_info::success (vinfo); - else - { - /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */ - loop_vinfo = vect_analyze_loop (loop, &shared); - loop->aux = loop_vinfo; - } + /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */ + opt_loop_vec_info loop_vinfo = vect_analyze_loop (loop, &shared); + loop->aux = loop_vinfo; if (!loop_vinfo) if (dump_enabled_p ()) @@ -1083,8 +1119,7 @@ try_vectorize_loop_1 (hash_table *&simduid_to_vf_htab, return ret; } - /* Only count the original scalar loops. */ - if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo) && !dbg_cnt (vect_loop)) + if (!dbg_cnt (vect_loop)) { /* Free existing information if loop is analyzed with some assumptions. */ @@ -1093,62 +1128,22 @@ try_vectorize_loop_1 (hash_table *&simduid_to_vf_htab, return ret; } - if (loop_vectorized_call) - set_uid_loop_bbs (loop_vinfo, loop_vectorized_call); - - unsigned HOST_WIDE_INT bytes; - if (dump_enabled_p ()) - { - if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes)) - dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, - "loop vectorized using %wu byte vectors\n", bytes); - else - dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, - "loop vectorized using variable length vectors\n"); - } - - loop_p new_loop = vect_transform_loop (loop_vinfo, - loop_vectorized_call); (*num_vectorized_loops)++; - /* Now that the loop has been vectorized, allow it to be unrolled - etc. */ - loop->force_vectorize = false; - - if (loop->simduid) - { - simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf); - if (!simduid_to_vf_htab) - simduid_to_vf_htab = new hash_table (15); - simduid_to_vf_data->simduid = DECL_UID (loop->simduid); - simduid_to_vf_data->vf = loop_vinfo->vectorization_factor; - *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT) - = simduid_to_vf_data; - } + /* Transform LOOP and its epilogues. */ + vect_transform_loops (simduid_to_vf_htab, loop, loop_vectorized_call); if (loop_vectorized_call) { fold_loop_internal_call (loop_vectorized_call, boolean_true_node); - loop_vectorized_call = NULL; ret |= TODO_cleanup_cfg; } if (loop_dist_alias_call) { tree value = gimple_call_arg (loop_dist_alias_call, 1); fold_loop_internal_call (loop_dist_alias_call, value); - loop_dist_alias_call = NULL; ret |= TODO_cleanup_cfg; } - /* Epilogue of vectorized loop must be vectorized too. */ - if (new_loop) - { - /* Don't include vectorized epilogues in the "vectorized loops" count. - */ - unsigned dont_count = *num_vectorized_loops; - ret |= try_vectorize_loop_1 (simduid_to_vf_htab, &dont_count, - new_loop, NULL, NULL); - } - return ret; }