From patchwork Tue Aug 9 12:00:32 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: =?utf-8?q?Martin_Li=C5=A1ka?= X-Patchwork-Id: 56618 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id E7381385356F for ; Tue, 9 Aug 2022 12:01:43 +0000 (GMT) X-Original-To: gcc-patches@gcc.gnu.org Delivered-To: gcc-patches@gcc.gnu.org Received: from smtp-out1.suse.de (smtp-out1.suse.de [IPv6:2001:67c:2178:6::1c]) by sourceware.org (Postfix) with ESMTPS id 9777A385AC1D for ; Tue, 9 Aug 2022 12:00:33 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org 9777A385AC1D Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=suse.cz Authentication-Results: sourceware.org; spf=fail smtp.mailfrom=suse.cz Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by smtp-out1.suse.de (Postfix) with ESMTPS id 8BBE733E5F for ; Tue, 9 Aug 2022 12:00:32 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=suse.cz; s=susede2_rsa; t=1660046432; h=from:from:reply-to:date:date:message-id:message-id:to:to:cc: mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=nKsCSQszriKqwC8t6tQGuehtY4JnoGnoqOrCvAXdNC4=; b=3Rw780sqoE14v+L+wgnSXH2v68H7DyLCIIFKyX66N+kDWCvKIvc0/4uFzDnkPx89TqWXez PrR/92TVIGwjmUyDnidBugoMBaEZKbGi6y7gxsl9pe9KOFNEiAvjzEyIZ5efXffy1IowAv 0ahMlvS2lypVbp/98y23qrZuYp9WBdA= DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=suse.cz; s=susede2_ed25519; t=1660046432; h=from:from:reply-to:date:date:message-id:message-id:to:to:cc: mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding; bh=nKsCSQszriKqwC8t6tQGuehtY4JnoGnoqOrCvAXdNC4=; b=Af0XgPEkGWwP3XHmYcw3b6iEbKEf+Ur+D9MQzgGYUAoI0wJQsnmxlU97JZPXE7HOE71d0C Pm+G8e9pQRxCEDAQ== Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by imap2.suse-dmz.suse.de (Postfix) with ESMTPS id 62E9A13A9D for ; Tue, 9 Aug 2022 12:00:32 +0000 (UTC) Received: from dovecot-director2.suse.de ([192.168.254.65]) by imap2.suse-dmz.suse.de with ESMTPSA id jhwCF2BM8mLRAwAAMHmgww (envelope-from ) for ; Tue, 09 Aug 2022 12:00:32 +0000 Message-ID: <7b34d4a1-4f34-cffb-8b5f-eb3424ca1049@suse.cz> Date: Tue, 9 Aug 2022 14:00:32 +0200 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Thunderbird/102.1.0 From: =?utf-8?q?Martin_Li=C5=A1ka?= Subject: [PATCH 1/3] Factor out jobserver_active_p. To: gcc-patches@gcc.gnu.org Content-Language: en-US X-Spam-Status: No, score=-30.5 required=5.0 tests=BAYES_00, DKIM_INVALID, DKIM_SIGNED, GIT_PATCH_0, KAM_DMARC_STATUS, KAM_SHORT, SPF_HELO_NONE, SPF_SOFTFAIL, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org X-BeenThere: gcc-patches@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-patches mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: gcc-patches-bounces+patchwork=sourceware.org@gcc.gnu.org Sender: "Gcc-patches" Patch can bootstrap on x86_64-linux-gnu and survives regression tests. Ready to be installed? Thanks, Martin gcc/ChangeLog: * gcc.cc (driver::detect_jobserver): Remove and move to jobserver.h. * lto-wrapper.cc (jobserver_active_p): Likewise. (run_gcc): Likewise. * jobserver.h: New file. --- gcc/gcc.cc | 36 +++----------------- gcc/jobserver.h | 85 ++++++++++++++++++++++++++++++++++++++++++++++ gcc/lto-wrapper.cc | 43 +++++------------------ 3 files changed, 97 insertions(+), 67 deletions(-) create mode 100644 gcc/jobserver.h diff --git a/gcc/gcc.cc b/gcc/gcc.cc index 5cbb38560b2..69fbd293eaa 100644 --- a/gcc/gcc.cc +++ b/gcc/gcc.cc @@ -43,6 +43,7 @@ compilation is specified by a string called a "spec". */ #include "opts.h" #include "filenames.h" #include "spellcheck.h" +#include "jobserver.h" @@ -9178,38 +9179,9 @@ driver::final_actions () const void driver::detect_jobserver () const { - /* Detect jobserver and drop it if it's not working. */ - const char *makeflags = env.get ("MAKEFLAGS"); - if (makeflags != NULL) - { - const char *needle = "--jobserver-auth="; - const char *n = strstr (makeflags, needle); - if (n != NULL) - { - int rfd = -1; - int wfd = -1; - - bool jobserver - = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2 - && rfd > 0 - && wfd > 0 - && is_valid_fd (rfd) - && is_valid_fd (wfd)); - - /* Drop the jobserver if it's not working now. */ - if (!jobserver) - { - unsigned offset = n - makeflags; - char *dup = xstrdup (makeflags); - dup[offset] = '\0'; - - const char *space = strchr (makeflags + offset, ' '); - if (space != NULL) - strcpy (dup + offset, space); - xputenv (concat ("MAKEFLAGS=", dup, NULL)); - } - } - } + jobserver_info jinfo; + if (!jinfo.is_active && !jinfo.skipped_makeflags.empty ()) + xputenv (jinfo.skipped_makeflags.c_str ()); } /* Determine what the exit code of the driver should be. */ diff --git a/gcc/jobserver.h b/gcc/jobserver.h new file mode 100644 index 00000000000..85453dd3c79 --- /dev/null +++ b/gcc/jobserver.h @@ -0,0 +1,85 @@ +/* GNU make's jobserver related functionality. + Copyright (C) 2022 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with GCC; see the file COPYING3. If not see +. + +See dbgcnt.def for usage information. */ + +#ifndef GCC_JOBSERVER_H +#define GCC_JOBSERVER_H + +#include + +using namespace std; + +struct jobserver_info +{ + /* Default constructor. */ + jobserver_info (); + + /* Error message if there is a problem. */ + string error_msg = ""; + /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ + string skipped_makeflags = ""; + /* File descriptor for reading used for jobserver communication. */ + int rfd = -1; + /* File descriptor for writing used for jobserver communication. */ + int wfd = -1; + /* Return true if jobserver is active. */ + bool is_active = false; +}; + +jobserver_info::jobserver_info () +{ + /* Detect jobserver and drop it if it's not working. */ + string js_needle = "--jobserver-auth="; + + const char *envval = getenv ("MAKEFLAGS"); + if (envval != NULL) + { + string makeflags = envval; + size_t n = makeflags.rfind (js_needle); + if (n != string::npos) + { + if (sscanf (makeflags.c_str () + n + js_needle.size (), + "%d,%d", &rfd, &wfd) == 2 + && rfd > 0 + && wfd > 0 + && is_valid_fd (rfd) + && is_valid_fd (wfd)) + is_active = true; + else + { + string dup = makeflags.substr (0, n); + size_t pos = makeflags.find (' ', n); + if (pos != string::npos) + dup += makeflags.substr (pos); + skipped_makeflags = "MAKEFLAGS=" + dup; + error_msg + = "cannot access %<" + js_needle + "%> file descriptors"; + } + } + error_msg = "%<" + js_needle + "%> is not present in %"; + } + else + error_msg = "% environment variable is unset"; + + if (!error_msg.empty ()) + error_msg = "jobserver is not available: " + error_msg; +} + +#endif /* GCC_JOBSERVER_H */ diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc index 795ab74555c..9279958d055 100644 --- a/gcc/lto-wrapper.cc +++ b/gcc/lto-wrapper.cc @@ -49,6 +49,8 @@ along with GCC; see the file COPYING3. If not see #include "lto-section-names.h" #include "collect-utils.h" #include "opts-diagnostic.h" +#include "opt-suggestions.h" +#include "jobserver.h" /* Environment variable, used for passing the names of offload targets from GCC driver to lto-wrapper. */ @@ -1336,35 +1338,6 @@ init_num_threads (void) #endif } -/* Test and return reason why a jobserver cannot be detected. */ - -static const char * -jobserver_active_p (void) -{ - #define JS_PREFIX "jobserver is not available: " - #define JS_NEEDLE "--jobserver-auth=" - - const char *makeflags = getenv ("MAKEFLAGS"); - if (makeflags == NULL) - return JS_PREFIX "% environment variable is unset"; - - const char *n = strstr (makeflags, JS_NEEDLE); - if (n == NULL) - return JS_PREFIX "%<" JS_NEEDLE "%> is not present in %"; - - int rfd = -1; - int wfd = -1; - - if (sscanf (n + strlen (JS_NEEDLE), "%d,%d", &rfd, &wfd) == 2 - && rfd > 0 - && wfd > 0 - && is_valid_fd (rfd) - && is_valid_fd (wfd)) - return NULL; - else - return JS_PREFIX "cannot access %<" JS_NEEDLE "%> file descriptors"; -} - /* Print link to -flto documentation with a hint message. */ void @@ -1422,7 +1395,6 @@ run_gcc (unsigned argc, char *argv[]) bool jobserver_requested = false; int auto_parallel = 0; bool no_partition = false; - const char *jobserver_error = NULL; bool fdecoded_options_first = true; vec fdecoded_options; fdecoded_options.create (16); @@ -1653,14 +1625,14 @@ run_gcc (unsigned argc, char *argv[]) } else { - jobserver_error = jobserver_active_p (); - if (jobserver && jobserver_error != NULL) + jobserver_info jinfo; + if (jobserver && !jinfo.is_active) { /* Fall back to auto parallelism. */ jobserver = 0; auto_parallel = 1; } - else if (!jobserver && jobserver_error == NULL) + else if (!jobserver && jinfo.is_active) { parallel = 1; jobserver = 1; @@ -1971,9 +1943,10 @@ cont: if (nr > 1) { - if (jobserver_requested && jobserver_error != NULL) + jobserver_info jinfo; + if (jobserver_requested && !jinfo.is_active) { - warning (0, jobserver_error); + warning (0, jinfo.error_msg.c_str ()); print_lto_docs_link (); } else if (parallel == 0) From patchwork Tue Aug 9 12:00:37 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: =?utf-8?q?Martin_Li=C5=A1ka?= X-Patchwork-Id: 56616 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id 432F7385C311 for ; Tue, 9 Aug 2022 12:00:59 +0000 (GMT) X-Original-To: gcc-patches@gcc.gnu.org Delivered-To: gcc-patches@gcc.gnu.org Received: from smtp-out2.suse.de (smtp-out2.suse.de [IPv6:2001:67c:2178:6::1d]) by sourceware.org (Postfix) with ESMTPS id CA9F03856092 for ; Tue, 9 Aug 2022 12:00:38 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org CA9F03856092 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=suse.cz Authentication-Results: sourceware.org; spf=fail smtp.mailfrom=suse.cz Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by smtp-out2.suse.de (Postfix) with ESMTPS id D59F720134 for ; Tue, 9 Aug 2022 12:00:37 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=suse.cz; s=susede2_rsa; t=1660046437; h=from:from:reply-to:date:date:message-id:message-id:to:to:cc: mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=lytcVulcSuWjTH56SdhfrMSHGjfwvP+tvGKJl/RnQD4=; b=aX93UQB7ZnOAdnSNoviFTudkDe+e3svfZERyMPT/k13qLWjqdl2bf+YGotXCmFMUH5KmZz W7iyXonj0W92UjVlKfC+NN7sI8kh1EJtBXvbTfIgNqEu6+/h/O/k0q861i+BGEPN8c7Nqo pAreZhUGdNYFO/8OkujNdJyZF/8/Ppw= DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=suse.cz; s=susede2_ed25519; t=1660046437; h=from:from:reply-to:date:date:message-id:message-id:to:to:cc: mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=lytcVulcSuWjTH56SdhfrMSHGjfwvP+tvGKJl/RnQD4=; b=aeSfiDIM77L1me0bZlEDy1TxOXYWfhsdQ3RGqBXVM0yOEI89AXB6nyueS28dF8fpMw276t Hpw1OExTdbHt2hDQ== Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by imap2.suse-dmz.suse.de (Postfix) with ESMTPS id C810213A9D for ; Tue, 9 Aug 2022 12:00:37 +0000 (UTC) Received: from dovecot-director2.suse.de ([192.168.254.65]) by imap2.suse-dmz.suse.de with ESMTPSA id z0nKL2VM8mLhAwAAMHmgww (envelope-from ) for ; Tue, 09 Aug 2022 12:00:37 +0000 Message-ID: <62cc745a-052a-fbde-7a8e-bccacf249bb8@suse.cz> Date: Tue, 9 Aug 2022 14:00:37 +0200 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Thunderbird/102.1.0 From: =?utf-8?q?Martin_Li=C5=A1ka?= Subject: [PATCH 2/3] lto: support --jobserver-style=fifo for recent GNU make To: gcc-patches@gcc.gnu.org References: <95d2ec7222d50c8ca1e6535cb8c272d47abbb08d.1660046383.git.mliska@suse.cz> Content-Language: en-US In-Reply-To: <95d2ec7222d50c8ca1e6535cb8c272d47abbb08d.1660046383.git.mliska@suse.cz> X-Spam-Status: No, score=-11.7 required=5.0 tests=BAYES_00, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, GIT_PATCH_0, SPF_HELO_NONE, SPF_SOFTFAIL, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org X-BeenThere: gcc-patches@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-patches mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: gcc-patches-bounces+patchwork=sourceware.org@gcc.gnu.org Sender: "Gcc-patches" Patch can bootstrap on x86_64-linux-gnu and survives regression tests. Ready to be installed? Thanks, Martin gcc/ChangeLog: * jobserver.h (jobserver_info::jobserver_info): Parse FIFO format of --jobserver-auth. --- gcc/jobserver.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/gcc/jobserver.h b/gcc/jobserver.h index 85453dd3c79..856e326ddfc 100644 --- a/gcc/jobserver.h +++ b/gcc/jobserver.h @@ -39,14 +39,22 @@ struct jobserver_info int rfd = -1; /* File descriptor for writing used for jobserver communication. */ int wfd = -1; + /* Named pipe path. */ + string pipe_path = ""; /* Return true if jobserver is active. */ bool is_active = false; }; jobserver_info::jobserver_info () { + /* Traditionally, GNU make uses opened pipes for jobserver-auth, + e.g. --jobserver-auth=3,4. + Starting with GNU make 4.4, one can use --jobserver-style=fifo + and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta. */ + /* Detect jobserver and drop it if it's not working. */ string js_needle = "--jobserver-auth="; + string fifo_prefix = "fifo:"; const char *envval = getenv ("MAKEFLAGS"); if (envval != NULL) @@ -55,8 +63,15 @@ jobserver_info::jobserver_info () size_t n = makeflags.rfind (js_needle); if (n != string::npos) { - if (sscanf (makeflags.c_str () + n + js_needle.size (), - "%d,%d", &rfd, &wfd) == 2 + string ending = makeflags.substr (n + js_needle.size ()); + if (ending.find (fifo_prefix) == 0) + { + ending = ending.substr (fifo_prefix.size ()); + pipe_path = ending.substr (0, ending.find (' ')); + is_active = true; + } + else if (sscanf (makeflags.c_str () + n + js_needle.size (), + "%d,%d", &rfd, &wfd) == 2 && rfd > 0 && wfd > 0 && is_valid_fd (rfd) From patchwork Tue Aug 9 12:00:42 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: =?utf-8?q?Martin_Li=C5=A1ka?= X-Patchwork-Id: 56617 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id D67B638560B6 for ; Tue, 9 Aug 2022 12:01:01 +0000 (GMT) X-Original-To: gcc-patches@gcc.gnu.org Delivered-To: gcc-patches@gcc.gnu.org Received: from smtp-out2.suse.de (smtp-out2.suse.de [195.135.220.29]) by sourceware.org (Postfix) with ESMTPS id B0FD5385AC1D for ; Tue, 9 Aug 2022 12:00:44 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org B0FD5385AC1D Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=suse.cz Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=suse.cz Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by smtp-out2.suse.de (Postfix) with ESMTPS id 573881FDB6 for ; Tue, 9 Aug 2022 12:00:43 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=suse.cz; s=susede2_rsa; t=1660046443; h=from:from:reply-to:date:date:message-id:message-id:to:to:cc: mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=8KzaMSGjc5p6+pNghtiwFKaJVf/6DC9367rxYvckkG0=; b=zXOuTTLYl4PUY+aCgw7sPKis0J2Yf4r1KngDVD/Vt70NlpVrTQlvQVVSeZjpnb9xQA1ywe +JUg3MY91v43yIG3uSpVDHWLFHgMCCmJ0OOkOjqrTmPCPWb/qUm2PZ/9tCplNnYLAvA+bi QNJNScczU3BN8g0gj/Qu3ijkEOpZcpg= DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=suse.cz; s=susede2_ed25519; t=1660046443; h=from:from:reply-to:date:date:message-id:message-id:to:to:cc: mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=8KzaMSGjc5p6+pNghtiwFKaJVf/6DC9367rxYvckkG0=; b=YalzVI1igiAioDh8IYAFtCUdNMTb9b1qvhx2QNZerH3vcYuYHlIp27SqKohCr6CuWvcFij 2z/IctDqLl98ssBw== Received: from imap2.suse-dmz.suse.de (imap2.suse-dmz.suse.de [192.168.254.74]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-521) server-digest SHA512) (No client certificate requested) by imap2.suse-dmz.suse.de (Postfix) with ESMTPS id 49D5D13A9D for ; Tue, 9 Aug 2022 12:00:43 +0000 (UTC) Received: from dovecot-director2.suse.de ([192.168.254.65]) by imap2.suse-dmz.suse.de with ESMTPSA id gsSJEGtM8mL0AwAAMHmgww (envelope-from ) for ; Tue, 09 Aug 2022 12:00:43 +0000 Message-ID: Date: Tue, 9 Aug 2022 14:00:42 +0200 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Thunderbird/102.1.0 From: =?utf-8?q?Martin_Li=C5=A1ka?= Subject: [PATCH 3/3] lto: respect jobserver in parallel WPA streaming To: gcc-patches@gcc.gnu.org References: <95d2ec7222d50c8ca1e6535cb8c272d47abbb08d.1660046383.git.mliska@suse.cz> <7ad7af4bb978bffdbc220c754002c667df1b0209.1660046383.git.mliska@suse.cz> Content-Language: en-US In-Reply-To: <7ad7af4bb978bffdbc220c754002c667df1b0209.1660046383.git.mliska@suse.cz> X-Spam-Status: No, score=-11.9 required=5.0 tests=BAYES_00, DKIM_SIGNED, DKIM_VALID, DKIM_VALID_AU, DKIM_VALID_EF, GIT_PATCH_0, SPF_HELO_NONE, SPF_PASS, TXREP, T_SCC_BODY_TEXT_LINE autolearn=ham autolearn_force=no version=3.4.6 X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on server2.sourceware.org X-BeenThere: gcc-patches@gcc.gnu.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Gcc-patches mailing list List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: gcc-patches-bounces+patchwork=sourceware.org@gcc.gnu.org Sender: "Gcc-patches" Patch can bootstrap on x86_64-linux-gnu and survives regression tests. Ready to be installed? Thanks, Martin PR lto/106328 gcc/ChangeLog: * jobserver.h (struct jobserver_info): Add pipefd. (jobserver_info::connect): New. (jobserver_info::disconnect): Likewise. (jobserver_info::get_token): Likewise. (jobserver_info::return_token): Likewise. gcc/lto/ChangeLog: * lto.cc (wait_for_child): Decrement nruns once a process finishes. (stream_out_partitions): Use job server if active. (do_whole_program_analysis): Likewise. --- gcc/jobserver.h | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ gcc/lto/lto.cc | 55 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/gcc/jobserver.h b/gcc/jobserver.h index 856e326ddfc..2a7dc9f4113 100644 --- a/gcc/jobserver.h +++ b/gcc/jobserver.h @@ -31,6 +31,18 @@ struct jobserver_info /* Default constructor. */ jobserver_info (); + /* Connect to the server. */ + void connect (); + + /* Disconnect from the server. */ + void disconnect (); + + /* Get token from the server. */ + bool get_token (); + + /* Return token to the server. */ + void return_token (); + /* Error message if there is a problem. */ string error_msg = ""; /* Skipped MAKEFLAGS where --jobserver-auth is skipped. */ @@ -41,6 +53,8 @@ struct jobserver_info int wfd = -1; /* Named pipe path. */ string pipe_path = ""; + /* Pipe file descriptor. */ + int pipefd = -1; /* Return true if jobserver is active. */ bool is_active = false; }; @@ -97,4 +111,44 @@ jobserver_info::jobserver_info () error_msg = "jobserver is not available: " + error_msg; } +void +jobserver_info::connect () +{ + if (!pipe_path.empty ()) + pipefd = open (pipe_path.c_str (), O_RDWR); +} + +void +jobserver_info::disconnect () +{ + if (!pipe_path.empty ()) + { + gcc_assert (close (pipefd) == 0); + pipefd = -1; + } +} + +bool +jobserver_info::get_token () +{ + int fd = pipe_path.empty () ? rfd : pipefd; + char c; + unsigned n = read (fd, &c, 1); + if (n != 1) + { + gcc_assert (errno == EAGAIN); + return false; + } + else + return true; +} + +void +jobserver_info::return_token () +{ + int fd = pipe_path.empty () ? wfd : pipefd; + char c = 'G'; + gcc_assert (write (fd, &c, 1) == 1); +} + #endif /* GCC_JOBSERVER_H */ diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc index 31b0c1862f7..56266195ead 100644 --- a/gcc/lto/lto.cc +++ b/gcc/lto/lto.cc @@ -54,11 +54,17 @@ along with GCC; see the file COPYING3. If not see #include "attribs.h" #include "builtins.h" #include "lto-common.h" - +#include "jobserver.h" /* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver. */ static int lto_parallelism; +/* Number of active WPA streaming processes. */ +static int nruns = 0; + +/* GNU make's jobserver info. */ +static jobserver_info *jinfo = NULL; + /* Return true when NODE has a clone that is analyzed (i.e. we need to load its body even if the node itself is not needed). */ @@ -205,6 +211,12 @@ wait_for_child () "streaming subprocess was killed by signal"); } while (!WIFEXITED (status) && !WIFSIGNALED (status)); + + --nruns; + + /* Return token to the jobserver if active. */ + if (jinfo != NULL && jinfo->is_active) + jinfo->return_token (); } #endif @@ -228,25 +240,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, bool ARG_UNUSED (last)) { #ifdef HAVE_WORKING_FORK - static int nruns; - if (lto_parallelism <= 1) { stream_out_partitions_1 (temp_filename, blen, min, max); return; } - /* Do not run more than LTO_PARALLELISM streamings - FIXME: we ignore limits on jobserver. */ if (lto_parallelism > 0 && nruns >= lto_parallelism) - { - wait_for_child (); - nruns --; - } + wait_for_child (); + /* If this is not the last parallel partition, execute new streaming process. */ if (!last) { + if (jinfo != NULL && jinfo->is_active) + while (true) + { + if (jinfo->get_token ()) + break; + if (nruns > 0) + wait_for_child (); + else + { + /* There are no free tokens, lets do the job outselves. */ + stream_out_partitions_1 (temp_filename, blen, min, max); + asm_nodes_output = true; + return; + } + } + pid_t cpid = fork (); if (!cpid) @@ -264,10 +286,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max, /* Last partition; stream it and wait for all children to die. */ else { - int i; stream_out_partitions_1 (temp_filename, blen, min, max); - for (i = 0; i < nruns; i++) + while (nruns > 0) wait_for_child (); + + if (jinfo != NULL && jinfo->is_active) + jinfo->disconnect (); } asm_nodes_output = true; #else @@ -460,9 +484,14 @@ do_whole_program_analysis (void) lto_parallelism = 1; - /* TODO: jobserver communication is not supported, yet. */ if (!strcmp (flag_wpa, "jobserver")) - lto_parallelism = param_max_lto_streaming_parallelism; + { + jinfo = new jobserver_info (); + if (jinfo->is_active) + jinfo->connect (); + + lto_parallelism = param_max_lto_streaming_parallelism; + } else { lto_parallelism = atoi (flag_wpa);