@@ -111,6 +111,9 @@ static const riscv_implied_info_t riscv_implied_info[] =
{"zfinx", "zicsr"},
{"zdinx", "zicsr"},
+ {"zicfiss", "zicsr"},
+ {"zicfiss", "zimop"},
+
{"zk", "zkn"},
{"zk", "zkr"},
{"zk", "zkt"},
@@ -325,6 +328,8 @@ static const struct riscv_ext_version riscv_ext_version_table[] =
{"zicclsm", ISA_SPEC_CLASS_NONE, 1, 0},
{"ziccrse", ISA_SPEC_CLASS_NONE, 1, 0},
+ {"zicfiss", ISA_SPEC_CLASS_NONE, 1, 0},
+
{"zimop", ISA_SPEC_CLASS_NONE, 1, 0},
{"zcmop", ISA_SPEC_CLASS_NONE, 1, 0},
@@ -1647,6 +1652,8 @@ static const riscv_ext_flag_table_t riscv_ext_flag_table[] =
RISCV_EXT_FLAG_ENTRY ("zicbop", x_riscv_zicmo_subext, MASK_ZICBOP),
RISCV_EXT_FLAG_ENTRY ("zic64b", x_riscv_zicmo_subext, MASK_ZIC64B),
+ RISCV_EXT_FLAG_ENTRY ("zicfiss", x_riscv_zi_subext, MASK_ZICFISS),
+
RISCV_EXT_FLAG_ENTRY ("zimop", x_riscv_mop_subext, MASK_ZIMOP),
RISCV_EXT_FLAG_ENTRY ("zcmop", x_riscv_mop_subext, MASK_ZCMOP),
@@ -679,3 +679,9 @@
return (riscv_symbolic_constant_p (op, &type)
&& type == SYMBOL_PCREL);
})
+
+;; Shadow stack operands only allow x1, x5 registers
+(define_predicate "x1x5_operand"
+ (and (match_operand 0 "register_operand")
+ (match_test "REGNO (op) == RETURN_ADDR_REGNUM
+ || REGNO (op) == T0_REGNUM")))
@@ -7496,6 +7496,9 @@ riscv_save_reg_p (unsigned int regno)
if (regno == GP_REGNUM || regno == THREAD_POINTER_REGNUM)
return false;
+ if (regno == RETURN_ADDR_REGNUM && TARGET_ZICFISS)
+ return true;
+
/* We must save every register used in this function. If this is not a
leaf function, then we must save all temporary registers. */
if (df_regs_ever_live_p (regno)
@@ -8049,7 +8052,7 @@ riscv_is_eh_return_data_register (unsigned int regno)
static void
riscv_for_each_saved_reg (poly_int64 sp_offset, riscv_save_restore_fn fn,
- bool epilogue, bool maybe_eh_return)
+ bool epilogue, bool maybe_eh_return, bool sibcall_p)
{
HOST_WIDE_INT offset, first_fp_offset;
unsigned int regno, num_masked_fp = 0;
@@ -8135,7 +8138,14 @@ riscv_for_each_saved_reg (poly_int64 sp_offset, riscv_save_restore_fn fn,
}
}
- riscv_save_restore_reg (word_mode, regno, offset, fn);
+ if (need_shadow_stack_push_pop_p () && epilogue && !sibcall_p
+ && !(maybe_eh_return && crtl->calls_eh_return)
+ && (regno == RETURN_ADDR_REGNUM)
+ && !cfun->machine->interrupt_handler_p)
+ riscv_save_restore_reg (word_mode, RISCV_PROLOGUE_TEMP_REGNUM,
+ offset, fn);
+ else
+ riscv_save_restore_reg (word_mode, regno, offset, fn);
}
/* This loop must iterate over the same space as its companion in
@@ -8729,6 +8739,9 @@ riscv_expand_prologue (void)
if (cfun->machine->naked_p)
return;
+ if (need_shadow_stack_push_pop_p ())
+ emit_insn (gen_sspush (Pmode, gen_rtx_REG (Pmode, RETURN_ADDR_REGNUM)));
+
/* prefer multi-push to save-restore libcall. */
if (riscv_use_multi_push (frame))
{
@@ -8772,7 +8785,7 @@ riscv_expand_prologue (void)
= get_multi_push_fpr_mask (multi_push_additional / UNITS_PER_WORD);
frame->fmask &= mask_fprs_push;
riscv_for_each_saved_reg (remaining_size, riscv_save_reg, false,
- false);
+ false, false);
frame->fmask = fmask & ~mask_fprs_push; /* mask for the rest FPRs. */
}
}
@@ -8823,7 +8836,8 @@ riscv_expand_prologue (void)
GEN_INT (-step1));
RTX_FRAME_RELATED_P (emit_insn (insn)) = 1;
}
- riscv_for_each_saved_reg (remaining_size, riscv_save_reg, false, false);
+ riscv_for_each_saved_reg (remaining_size, riscv_save_reg,
+ false, false, false);
}
/* Undo the above fib. */
@@ -8985,6 +8999,7 @@ riscv_expand_epilogue (int style)
= use_multi_pop ? frame->multi_push_adj_base + frame->multi_push_adj_addi
: 0;
rtx ra = gen_rtx_REG (Pmode, RETURN_ADDR_REGNUM);
+ rtx t0 = gen_rtx_REG (Pmode, RISCV_PROLOGUE_TEMP_REGNUM);
unsigned th_int_mask = 0;
rtx insn;
@@ -9194,7 +9209,8 @@ riscv_expand_epilogue (int style)
riscv_for_each_saved_v_reg (step2, riscv_restore_reg, false);
riscv_for_each_saved_reg (frame->total_size - step2 - libcall_size
- multipop_size,
- riscv_restore_reg, true, style == EXCEPTION_RETURN);
+ riscv_restore_reg, true, style == EXCEPTION_RETURN,
+ style == SIBCALL_RETURN);
if (th_int_mask && TH_INT_INTERRUPT (cfun))
{
@@ -9234,7 +9250,8 @@ riscv_expand_epilogue (int style)
riscv_for_each_saved_reg (frame->total_size - libcall_size
- multipop_size,
riscv_restore_reg, true,
- style == EXCEPTION_RETURN);
+ style == EXCEPTION_RETURN, false);
+
/* Undo the above fib. */
frame->mask = mask;
frame->fmask = fmask;
@@ -9259,6 +9276,17 @@ riscv_expand_epilogue (int style)
emit_insn (gen_add3_insn (stack_pointer_rtx, stack_pointer_rtx,
EH_RETURN_STACKADJ_RTX));
+ if (need_shadow_stack_push_pop_p ()
+ && !((style == EXCEPTION_RETURN) && crtl->calls_eh_return))
+ {
+ if (BITSET_P (cfun->machine->frame.mask, RETURN_ADDR_REGNUM)
+ && style != SIBCALL_RETURN
+ && !cfun->machine->interrupt_handler_p)
+ emit_insn (gen_sspopchk (Pmode, t0));
+ else
+ emit_insn (gen_sspopchk (Pmode, ra));
+ }
+
/* Return from interrupt. */
if (cfun->machine->interrupt_handler_p)
{
@@ -9276,7 +9304,15 @@ riscv_expand_epilogue (int style)
emit_jump_insn (gen_riscv_uret ());
}
else if (style != SIBCALL_RETURN)
- emit_jump_insn (gen_simple_return_internal (ra));
+ {
+ if (need_shadow_stack_push_pop_p ()
+ && !((style == EXCEPTION_RETURN) && crtl->calls_eh_return)
+ && BITSET_P (cfun->machine->frame.mask, RETURN_ADDR_REGNUM)
+ && !cfun->machine->interrupt_handler_p)
+ emit_jump_insn (gen_simple_return_internal (t0));
+ else
+ emit_jump_insn (gen_simple_return_internal (ra));
+ }
}
/* Implement EPILOGUE_USES. */
@@ -9473,7 +9509,8 @@ bool
riscv_can_use_return_insn (void)
{
return (reload_completed && known_eq (cfun->machine->frame.total_size, 0)
- && ! cfun->machine->interrupt_handler_p);
+ && ! cfun->machine->interrupt_handler_p
+ && ! need_shadow_stack_push_pop_p ());
}
/* Given that there exists at least one variable that is set (produced)
@@ -13786,6 +13823,11 @@ expand_reversed_crc_using_clmul (scalar_mode crc_mode, scalar_mode data_mode,
riscv_emit_move (operands[0], gen_lowpart (crc_mode, a0));
}
+bool need_shadow_stack_push_pop_p ()
+{
+ return TARGET_ZICFISS && riscv_save_return_addr_reg_p ();
+}
+
/* Initialize the GCC target structure. */
#undef TARGET_ASM_ALIGNED_HI_OP
#define TARGET_ASM_ALIGNED_HI_OP "\t.half\t"
@@ -1189,6 +1189,7 @@ extern poly_int64 riscv_v_adjust_nunits (enum machine_mode, int);
extern poly_int64 riscv_v_adjust_nunits (machine_mode, bool, int, int);
extern poly_int64 riscv_v_adjust_precision (enum machine_mode, int);
extern poly_int64 riscv_v_adjust_bytesize (enum machine_mode, int);
+extern bool need_shadow_stack_push_pop_p ();
/* The number of bits and bytes in a RVV vector. */
#define BITS_PER_RISCV_VECTOR (poly_uint16 (riscv_vector_chunks * riscv_bytes_per_vector_chunk * 8))
#define BYTES_PER_RISCV_VECTOR (poly_uint16 (riscv_vector_chunks * riscv_bytes_per_vector_chunk))
@@ -99,6 +99,12 @@
;; CRC unspecs
UNSPEC_CRC
UNSPEC_CRC_REV
+
+ ;; ZICFISS
+ UNSPECV_SSPUSH
+ UNSPECV_SSPOPCHK
+ UNSPECV_SSRDP
+ UNSPECV_SSP
])
(define_c_enum "unspecv" [
@@ -4116,6 +4122,30 @@
(set_attr "length" "0")]
)
+(define_expand "save_stack_nonlocal"
+ [(set (match_operand 0 "memory_operand")
+ (match_operand 1 "register_operand"))]
+ ""
+{
+ rtx stack_slot;
+
+ if (need_shadow_stack_push_pop_p ())
+ {
+ /* Copy shadow stack pointer to the first slot
+ and stack pointer to the second slot. */
+ rtx ssp_slot = adjust_address (operands[0], word_mode, 0);
+ stack_slot = adjust_address (operands[0], Pmode, UNITS_PER_WORD);
+
+ rtx reg_ssp = force_reg (word_mode, const0_rtx);
+ emit_insn (gen_ssrdp (word_mode, reg_ssp));
+ emit_move_insn (ssp_slot, reg_ssp);
+ }
+ else
+ stack_slot = adjust_address (operands[0], Pmode, 0);
+ emit_move_insn (stack_slot, operands[1]);
+ DONE;
+})
+
;; This fixes a failure with gcc.c-torture/execute/pr64242.c at -O2 for a
;; 32-bit target when using -mtune=sifive-7-series. The first sched pass
;; runs before register elimination, and we have a non-obvious dependency
@@ -4126,7 +4156,70 @@
(match_operand 1 "memory_operand")]
""
{
- emit_move_insn (operands[0], operands[1]);
+ rtx stack_slot;
+
+ if (need_shadow_stack_push_pop_p ())
+ {
+ rtx t0 = gen_rtx_REG (Pmode, RISCV_PROLOGUE_TEMP_REGNUM);
+ /* Restore shadow stack pointer from the first slot
+ and stack pointer from the second slot. */
+ rtx ssp_slot = adjust_address (operands[1], word_mode, 0);
+ stack_slot = adjust_address (operands[1], Pmode, UNITS_PER_WORD);
+
+ /* Get the current shadow stack pointer. */
+ rtx cur_ssp = force_reg (word_mode, const0_rtx);
+ emit_insn (gen_ssrdp (word_mode, cur_ssp));
+
+ /* Compare and jump over adjustment code. */
+ rtx noadj_label = gen_label_rtx ();
+ emit_cmp_and_jump_insns (cur_ssp, const0_rtx, EQ, NULL_RTX,
+ word_mode, 1, noadj_label);
+
+ rtx loop_label = gen_label_rtx ();
+ emit_label (loop_label);
+ LABEL_NUSES (loop_label) = 1;
+
+ /* Check if current ssp less than jump buffer ssp,
+ so no loop is needed. */
+ emit_cmp_and_jump_insns (ssp_slot, cur_ssp, LE, NULL_RTX,
+ ptr_mode, 1, noadj_label);
+
+ /* Advance by a maximum of 4K at a time to avoid unwinding
+ past bounds of the shadow stack. */
+ rtx reg_4096 = force_reg (word_mode, GEN_INT (4096));
+ rtx cmp_ssp = gen_reg_rtx (word_mode);
+ cmp_ssp = expand_simple_binop (ptr_mode, MINUS,
+ ssp_slot, cur_ssp,
+ cmp_ssp, 1, OPTAB_DIRECT);
+
+ /* Update curr_ssp from jump buffer ssp. */
+ emit_move_insn (cur_ssp, ssp_slot);
+ emit_insn (gen_write_ssp (word_mode, cur_ssp));
+ emit_jump_insn (gen_jump (loop_label));
+ emit_barrier ();
+
+ /* Adjust the ssp in a loop. */
+ rtx cmp_4k_label = gen_label_rtx ();
+ emit_label (cmp_4k_label);
+ LABEL_NUSES (cmp_4k_label) = 1;
+
+ /* Add 4k for curr_ssp. */
+ cur_ssp = expand_simple_binop (ptr_mode, PLUS,
+ cur_ssp, reg_4096,
+ cur_ssp, 1, OPTAB_DIRECT);
+ emit_insn (gen_write_ssp (word_mode, cur_ssp));
+ emit_insn (gen_sspush (Pmode, t0));
+ emit_insn (gen_sspopchk (Pmode, t0));
+ emit_jump_insn (gen_jump (loop_label));
+ emit_barrier ();
+
+ emit_label (noadj_label);
+ LABEL_NUSES (noadj_label) = 1;
+ }
+ else
+ stack_slot = adjust_address (operands[1], Pmode, 0);
+
+ emit_move_insn (operands[0], stack_slot);
/* Prevent the following hard fp restore from being moved before the move
insn above which uses a copy of the soft fp reg. */
emit_clobber (gen_rtx_MEM (BLKmode, hard_frame_pointer_rtx));
@@ -4594,6 +4687,36 @@
}"
[(set_attr "type" "arith")])
+;; Shadow stack
+
+(define_insn "@sspush<mode>"
+ [(unspec_volatile [(match_operand:P 0 "x1x5_operand" "r")] UNSPECV_SSPUSH)]
+ "TARGET_ZICFISS"
+ "sspush\t%0"
+ [(set_attr "type" "arith")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "@sspopchk<mode>"
+ [(unspec_volatile [(match_operand:P 0 "x1x5_operand" "r")] UNSPECV_SSPOPCHK)]
+ "TARGET_ZICFISS"
+ "sspopchk\t%0"
+ [(set_attr "type" "arith")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "@ssrdp<mode>"
+ [(set (match_operand:P 0 "register_operand" "=r")
+ (unspec_volatile [(const_int 0)] UNSPECV_SSRDP))]
+ "TARGET_ZICFISS"
+ "ssrdp\t%0"
+ [(set_attr "type" "arith")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "@write_ssp<mode>"
+ [(unspec_volatile [(match_operand:P 0 "register_operand" "r")] UNSPECV_SSP)]
+ "TARGET_ZICFISS"
+ "csrw\tssp, %0"
+ [(set_attr "type" "arith")
+ (set_attr "mode" "<MODE>")])
(include "bitmanip.md")
(include "crypto.md")
@@ -253,6 +253,8 @@ Mask(ZICCLSM) Var(riscv_zi_subext)
Mask(ZICCRSE) Var(riscv_zi_subext)
+Mask(ZICFISS) Var(riscv_zi_subext)
+
TargetVariable
int riscv_za_subext
new file mode 100644
@@ -0,0 +1,41 @@
+/* { dg-do compile { target { riscv64*-*-* } } } */
+/* { dg-options "-O2 -march=rv64gc_zicfiss -mabi=lp64d" } */
+/* { dg-skip-if "" { *-*-* } { "-O0" } } */
+struct ad {
+ void *ae;
+};
+struct af {
+ union {
+ int *ai;
+ int *aj;
+ struct ad *ak;
+ } u;
+ struct {
+ struct {
+ long al : 1;
+ long am : 1;
+ long : 21;
+ } b;
+ long i;
+ } s;
+};
+void fdes (struct af *, void *, long *);
+
+void foo (struct af *bv, long *bw) {
+ bw[0] = bw[1] = 0;
+ if (bv->s.b.al)
+ fdes (bv, bv->u.ak->ae, bw);
+ else if (bv->s.b.am) {
+ int **p = (int**)bv->u.aj;
+ for (; *p; ++p)
+ fdes (bv, *p, bw);
+ } else
+ fdes (bv, bv->u.ai, bw);
+}
+
+/* { dg-final { scan-assembler-times "ld\tt0" 1 } } */
+/* { dg-final { scan-assembler-times "sspopchk\tt0" 1 } } */
+/* { dg-final { scan-assembler-times "jr\tt0" 1 } } */
+/* { dg-final { scan-assembler-times "ld\tra" 2 } } */
+/* { dg-final { scan-assembler-times "sspopchk\tra" 2 } } */
+/* { dg-final { scan-assembler-times "tail" 2 } } */
new file mode 100644
@@ -0,0 +1,10 @@
+/* { dg-do compile { target { riscv64*-*-* } } } */
+/* { dg-options "-O0 -march=rv64gc_zicfiss -mabi=lp64d" } */
+
+void __attribute__ ((interrupt))
+foo (void)
+{
+}
+/* { dg-final { scan-assembler-times "sd\tra" 1 } } */
+/* { dg-final { scan-assembler-times "ld\tra" 1 } } */
+/* { dg-final { scan-assembler-times "sspopchk\tra" 1 } } */
@@ -19,6 +19,11 @@
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
+/* Unwind shadow stack. */
+#if defined(__riscv_zicfiss)
+# include "config/riscv/shadow-stack-unwind.h"
+#endif
+
#ifndef inhibit_libc
#include <signal.h>
new file mode 100644
@@ -0,0 +1,74 @@
+/* _Unwind_Frames_Extra with shadow stack.
+ Copyright (C) 2016-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
+<http://www.gnu.org/licenses/>. */
+
+#define LIBGCC2_UNITS_PER_WORD (__riscv_xlen / 8)
+
+/* Unwind the shadow stack for EH. */
+#undef _Unwind_Frames_Extra
+#define _Unwind_Frames_Extra(x) \
+ do \
+ { \
+ _Unwind_Word ssp = 0; \
+ asm volatile ("ssrdp %0" : "=r"(ssp)); \
+ if (ssp != 0) \
+ { \
+ _Unwind_Word tmp = (x); \
+ tmp = tmp * LIBGCC2_UNITS_PER_WORD; \
+ while (tmp > 4096) \
+ { \
+ ssp += 4096; \
+ tmp -= 4096; \
+ asm volatile ("csrw ssp, %0" :: "r"(ssp)); \
+ asm volatile ("sspush x5\n" \
+ "sspopchk x5" : : : "x5"); \
+ } \
+ \
+ if (tmp > 0) \
+ { \
+ ssp += tmp; \
+ asm volatile ("csrw ssp, %0" :: "r"(ssp)); \
+ } \
+ } \
+ } \
+ while (0)
+
+#undef _Unwind_Frames_Increment
+#define _Unwind_Frames_Increment(exc, context, frames) \
+ { \
+ frames++; \
+ if (exc->exception_class != 0 \
+ && _Unwind_GetIP (context) != 0 \
+ && !_Unwind_IsSignalFrame (context)) \
+ { \
+ _Unwind_Word ssp; \
+ asm volatile ("ssrdp %0" : "=r"(ssp)); \
+ if (ssp != 0) \
+ { \
+ ssp += LIBGCC2_UNITS_PER_WORD * frames; \
+ _Unwind_Word ra = *(_Unwind_Word *) ssp; \
+ if (ra != _Unwind_GetIP (context)) \
+ return _URC_FATAL_PHASE2_ERROR; \
+ } \
+ } \
+ }