@@ -116,7 +116,20 @@ (define_special_memory_constraint "RF"
"Buffer memory address to flat memory."
(and (match_code "mem")
(match_test "AS_FLAT_P (MEM_ADDR_SPACE (op))
- && gcn_flat_address_p (XEXP (op, 0), mode)")))
+ && gcn_flat_address_p (XEXP (op, 0), mode)
+ && !VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
+
+(define_special_memory_constraint "Rf"
+ "Buffer memory address to flat memory."
+ (and (match_code "mem")
+ (match_test "AS_FLAT_P (MEM_ADDR_SPACE (op))
+ && gcn_flat_address_p (XEXP (op, 0), mode)
+ && VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
+
+(define_constraint "Pf"
+ "Integer matching ADDR_SPACE_FLAT"
+ (and (match_code "const_int")
+ (match_test "AS_FLAT_P (ival)")))
(define_special_memory_constraint "RS"
"Buffer memory address to scalar flat memory."
@@ -127,7 +140,19 @@ (define_special_memory_constraint "RS"
(define_special_memory_constraint "RL"
"Buffer memory address to LDS memory."
(and (match_code "mem")
- (match_test "AS_LDS_P (MEM_ADDR_SPACE (op))")))
+ (match_test "AS_LDS_P (MEM_ADDR_SPACE (op))
+ && !VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
+
+(define_special_memory_constraint "Rl"
+ "Buffer memory address to LDS memory."
+ (and (match_code "mem")
+ (match_test "AS_LDS_P (MEM_ADDR_SPACE (op))
+ && VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
+
+(define_constraint "Pl"
+ "Integer matching ADDR_SPACE_LDS"
+ (and (match_code "const_int")
+ (match_test "AS_LDS_P (ival)")))
(define_special_memory_constraint "RG"
"Buffer memory address to GDS memory."
@@ -144,4 +169,22 @@ (define_special_memory_constraint "RM"
"Memory address to global (main) memory."
(and (match_code "mem")
(match_test "AS_GLOBAL_P (MEM_ADDR_SPACE (op))
- && gcn_global_address_p (XEXP (op, 0))")))
+ && gcn_global_address_p (XEXP (op, 0), GET_MODE (op))
+ && !VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
+
+(define_special_memory_constraint "Rm"
+ "Memory address to global (main) memory."
+ (and (match_code "mem")
+ (match_test "AS_GLOBAL_P (MEM_ADDR_SPACE (op))
+ && gcn_global_address_p (XEXP (op, 0), GET_MODE (op))
+ && VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
+
+(define_constraint "Pm"
+ "Integer matching ADDR_SPACE_GLOBAL"
+ (and (match_code "const_int")
+ (match_test "AS_GLOBAL_P (ival)")))
+
+(define_special_memory_constraint "RA"
+ "All memory types that use a scalar address."
+ (and (match_code "mem")
+ (match_test "!VECTOR_MODE_P (GET_MODE (XEXP (op, 0)))")))
@@ -37,14 +37,17 @@ extern char * gcn_expand_dpp_distribute_odd_insn (machine_mode, const char *,
extern void gcn_expand_epilogue ();
extern rtx gcn_expand_scaled_offsets (addr_space_t as, rtx base, rtx offsets,
rtx scale, bool unsigned_p, rtx exec);
+extern rtx gcn_gen_vector_mem (machine_mode mode, addr_space_t as,
+ rtx scalarbase, rtx vectoroffsets, rtx scale,
+ bool unsigned_p, bool volatile_p, rtx exec);
extern void gcn_expand_prologue ();
extern rtx gcn_expand_reduc_scalar (machine_mode, rtx, int);
extern rtx gcn_expand_scalar_to_vector_address (machine_mode, rtx, rtx, rtx);
extern void gcn_expand_vector_init (rtx, rtx);
-extern bool gcn_flat_address_p (rtx, machine_mode);
+extern bool gcn_flat_address_p (rtx, machine_mode, bool strict = false);
extern bool gcn_fp_constant_p (rtx, bool);
extern rtx gcn_gen_undef (machine_mode);
-extern bool gcn_global_address_p (rtx);
+extern bool gcn_global_address_p (rtx, machine_mode, bool strict = false);
extern tree gcn_goacc_adjust_private_decl (location_t, tree var, int level);
extern tree gcn_goacc_create_worker_broadcast_record (tree record_type,
bool sender,
@@ -67,6 +70,7 @@ extern bool gcn_inline_constant_p (rtx);
extern int gcn_inline_fp_constant_p (rtx, bool);
extern reg_class gcn_mode_code_base_reg_class (machine_mode, addr_space_t,
int, int);
+extern reg_class gcn_insn_base_reg_class (rtx_insn *);
extern rtx gcn_oacc_dim_pos (int dim);
extern rtx gcn_oacc_dim_size (int dim);
extern rtx gcn_operand_doublepart (machine_mode, rtx, int);
@@ -74,7 +78,7 @@ extern rtx gcn_operand_part (machine_mode, rtx, int);
extern bool gcn_regno_mode_code_ok_for_base_p (int, machine_mode,
addr_space_t, int, int);
extern reg_class gcn_regno_reg_class (int regno);
-extern bool gcn_scalar_flat_address_p (rtx);
+extern bool gcn_scalar_flat_address_p (rtx, bool strict = false);
extern bool gcn_scalar_flat_mem_p (rtx);
extern bool gcn_sgpr_move_p (rtx, rtx);
extern bool gcn_stepped_zero_int_parallel_p (rtx op, int step);
@@ -304,8 +304,6 @@ (define_subst_attr "exec_clobber" "vec_merge_with_clobber"
"" "_exec")
(define_subst_attr "exec_vcc" "vec_merge_with_vcc"
"" "_exec")
-(define_subst_attr "exec_scatter" "scatter_store"
- "" "_exec")
(define_subst "vec_merge"
[(set (match_operand:V_MOV 0)
@@ -345,24 +343,6 @@ (define_subst "vec_merge_with_vcc"
(and:DI (match_dup 3)
(reg:DI EXEC_REG)))])])
-(define_subst "scatter_store"
- [(set (mem:BLK (scratch))
- (unspec:BLK
- [(match_operand 0)
- (match_operand 1)
- (match_operand 2)
- (match_operand 3)]
- UNSPEC_SCATTER))]
- ""
- [(set (mem:BLK (scratch))
- (unspec:BLK
- [(match_dup 0)
- (match_dup 1)
- (match_dup 2)
- (match_dup 3)
- (match_operand:DI 4 "gcn_exec_reg_operand" "e")]
- UNSPEC_SCATTER))])
-
;; }}}
;; {{{ Vector moves
@@ -406,30 +386,31 @@ (define_expand "mov<mode>"
&& (!SUBREG_P (operands[1])
|| !MEM_P (SUBREG_REG (operands[1]))));
- if (MEM_P (operands[0]) && !lra_in_progress && !reload_completed)
+ if ((MEM_P (operands[0])
+ && VECTOR_MODE_P (GET_MODE (XEXP (operands[0], 0))))
+ || (MEM_P (operands[1])
+ && VECTOR_MODE_P (GET_MODE (XEXP (operands[1], 0)))))
+ /* Use default expand. */
+ ;
+ else if (MEM_P (operands[0]) && !lra_in_progress && !reload_completed)
{
operands[1] = force_reg (<MODE>mode, operands[1]);
rtx scratch = gen_rtx_SCRATCH (<VnDI>mode);
- rtx a = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[0]));
- rtx v = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[0]));
- rtx expr = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
- operands[0],
- scratch);
- emit_insn (gen_scatter<mode>_expr (expr, operands[1], a, v));
- DONE;
+ operands[0] = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
+ operands[0],
+ scratch);
}
else if (MEM_P (operands[1]) && !lra_in_progress && !reload_completed)
{
rtx scratch = gen_rtx_SCRATCH (<VnDI>mode);
- rtx a = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[1]));
- rtx v = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[1]));
- rtx expr = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
- operands[1],
- scratch);
- emit_insn (gen_gather<mode>_expr (operands[0], expr, a, v));
- DONE;
+ operands[1] = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
+ operands[1],
+ scratch);
}
- else if ((MEM_P (operands[0]) || MEM_P (operands[1])))
+ else if ((MEM_P (operands[0])
+ && !VECTOR_MODE_P (GET_MODE (XEXP (operands[0], 0))))
+ || (MEM_P (operands[1])
+ && !VECTOR_MODE_P (GET_MODE (XEXP (operands[1], 0)))))
{
gcc_assert (!reload_completed);
rtx scratch = gen_reg_rtx (<VnDI>mode);
@@ -448,88 +429,205 @@ (define_insn "mov<mode>_unspec"
[(set_attr "type" "unknown")
(set_attr "length" "0")])
-(define_insn "*mov<mode>"
+(define_insn "*mov<mode>_1reg"
[(set (match_operand:V_1REG 0 "nonimmediate_operand")
(match_operand:V_1REG 1 "general_operand"))]
+ "!MEM_P (operands[0]) || REG_P (operands[1])"
+ {@ [cons: =0, 1; attrs: type, length, cdna, xnack]
+ [v ,vA;vop1 ,4 ,* ,* ] v_mov_b32\t%0, %1
+ [v ,B ;vop1 ,8 ,* ,* ] ^
+ [v ,Rf;flat ,12,* ,off] flat_load%o1\t%0, %A1%O1\;s_waitcnt\t0
+ [&v ,Rf;flat ,12,* ,on ] ^
+ [^a ,Rf;flat ,12,cdna2,off] ^
+ [&^a,Rf;flat ,12,cdna2,on ] ^
+ [Rf ,v ;flat ,12,* ,* ] flat_store%s0\t%A0, %1%O0
+ [Rf ,a ;flat ,12,cdna2,* ] ^
+ [v ,Rm;flat ,12,* ,off] global_load%o1\t%0, %A1%O1\;s_waitcnt\tvmcnt(0)
+ [&v ,Rm;flat ,12,* ,on ] ^
+ [^a ,Rm;flat ,12,cdna2,off] ^
+ [&^a,Rm;flat ,12,cdna2,on ] ^
+ [Rm ,v ;flat ,12,* ,* ] global_store%s0\t%A0, %1%O0
+ [Rm ,a ;flat ,12,cdna2,* ] ^
+ [v ,Rl;ds ,12,* ,* ] ds_read%b1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+ [Rl ,v ;ds ,12,* ,* ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+ [v ,a ;vop3p_mai,8 ,* ,* ] v_accvgpr_read_b32\t%0, %1
+ [$a ,v ;vop3p_mai,8 ,* ,* ] v_accvgpr_write_b32\t%0, %1
+ [a ,a ;vop1 ,4 ,cdna2,* ] v_accvgpr_mov_b32\t%0, %1
+ })
+
+(define_insn "*mov<mode>_2reg"
+ [(set (match_operand:V_2REG 0 "nonimmediate_operand")
+ (match_operand:V_2REG 1 "general_operand"))]
+ "!MEM_P (operands[0]) || REG_P (operands[1])"
+ {@ [cons: =0, 1; attrs: type, length, cdna, xnack]
+ [v ,vDB;vmult,16,* ,* ] v_mov_b32\t%L0, %L1\;v_mov_b32\t%H0, %H1
+ [v ,a ;vmult,16,* ,* ] v_accvgpr_read_b32\t%L0, %L1\;v_accvgpr_read_b32\t%H0, %H1
+ [$a ,v ;vmult,16,* ,* ] v_accvgpr_write_b32\t%L0, %L1\;v_accvgpr_write_b32\t%H0, %H1
+ [a ,a ;vmult,8 ,cdna2,* ] v_accvgpr_mov_b32\t%L0, %L1\;v_accvgpr_mov_b32\t%H0, %H1
+ [v ,Rf ;flat ,12,* ,off] flat_load_dwordx2\t%0, %A1%O1\;s_waitcnt\t0
+ [&v ,Rf ;flat ,12,* ,on ] ^
+ [^a ,Rf ;flat ,12,cdna2,off] ^
+ [&^a,Rf ;flat ,12,cdna2,on ] ^
+ [Rf ,v ;flat ,12,* ,* ] flat_store_dwordx2\t%A0, %1%O0
+ [Rf ,a ;flat ,12,cdna2,* ] ^
+ [v ,Rm ;flat ,12,* ,off] global_load_dwordx2\t%0, %A1%O1\;s_waitcnt\tvmcnt(0)
+ [&v ,Rm ;flat ,12,* ,on ] ^
+ [^a ,Rm ;flat ,12,cdna2,off] ^
+ [&^a,Rm ;flat ,12,cdna2,on ] ^
+ [Rm ,v ;flat ,12,* ,* ] global_store_dwordx2\t%A0, %1%O0
+ [Rm ,a ;flat ,12,cdna2,* ] ^
+ [v ,Rl ;ds ,12,* ,* ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+ [Rl ,v ;ds ,12,* ,* ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+ })
+
+(define_insn "*mov<mode>_4reg"
+ [(set (match_operand:V_4REG 0 "nonimmediate_operand")
+ (match_operand:V_4REG 1 "general_operand"))]
+ "!MEM_P (operands[0]) || REG_P (operands[1])"
+ {@ [cons: =0, 1; attrs: type, length, cdna, xnack]
+ [v ,vDB;vmult,16,* ,* ] v_mov_b32\t%L0, %L1\; v_mov_b32\t%H0, %H1\; v_mov_b32\t%J0, %J1\; v_mov_b32\t%K0, %K1
+ [v ,a ;vmult,32,* ,* ] v_accvgpr_read_b32\t%L0, %L1\; v_accvgpr_read_b32\t%H0, %H1\; v_accvgpr_read_b32\t%J0, %J1\; v_accvgpr_read_b32\t%K0, %K1
+ [$a ,v ;vmult,32,* ,* ] v_accvgpr_write_b32\t%L0, %L1\;v_accvgpr_write_b32\t%H0, %H1\;v_accvgpr_write_b32\t%J0, %J1\;v_accvgpr_write_b32\t%K0, %K1
+ [a ,a ;vmult,32,cdna2,* ] v_accvgpr_mov_b32\t%L0, %L1\; v_accvgpr_mov_b32\t%H0, %H1\; v_accvgpr_mov_b32\t%J0, %J1\; v_accvgpr_mov_b32\t%K0, %K1
+ [v ,Rf ;flat ,12,* ,off] flat_load_dwordx4\t%0, %A1%O1\;s_waitcnt\t0
+ [&v ,Rf ;flat ,12,* ,on ] ^
+ [^a ,Rf ;flat ,12,cdna2,off] ^
+ [&^a,Rf ;flat ,12,cdna2,on ] ^
+ [Rf ,v ;flat ,12,* ,* ] flat_store_dwordx4\t%A0, %1%O0
+ [Rf ,a ;flat ,12,cdna2,* ] ^
+ [v ,Rm ;flat ,12,* ,off] global_load_dwordx4\t%0, %A1%O1\;s_waitcnt\tvmcnt(0)
+ [&v ,Rm ;flat ,12,* ,on ] ^
+ [^a ,Rm ;flat ,12,cdna2,off] ^
+ [&^a,Rm ;flat ,12,cdna2,on ] ^
+ [Rm ,v ;flat ,12,* ,* ] global_store_dwordx4\t%A0, %1%O0
+ [Rm ,a ;flat ,12,cdna2,* ] ^
+ })
+
+(define_expand "@mov<mode>_exec"
+ [(parallel [
+ (set (match_operand:V_MOV 0 "nonimmediate_operand")
+ (vec_merge:V_MOV
+ (match_operand:V_MOV 1 "general_operand")
+ (match_operand:V_MOV 2 "general_or_unspec_operand")
+ (match_operand:DI 3 "gcn_exec_operand")))
+ (clobber (match_scratch:<VnDI> 4))])]
""
- {@ [cons: =0, 1; attrs: type, length, cdna]
- [v ,vA;vop1 ,4,* ] v_mov_b32\t%0, %1
- [v ,B ;vop1 ,8,* ] ^
- [v ,a ;vop3p_mai,8,* ] v_accvgpr_read_b32\t%0, %1
- [$a ,v ;vop3p_mai,8,* ] v_accvgpr_write_b32\t%0, %1
- [a ,a ;vop1 ,4,cdna2] v_accvgpr_mov_b32\t%0, %1
+ {
+ if (MEM_P (operands[1]))
+ {
+ /* Masked loads need an insn that doesn't use an explicit MEM. */
+ rtx addr = XEXP (operands[1], 0);
+ rtx offset = const0_rtx;
+ if (GET_CODE (addr) == PLUS)
+ {
+ offset = XEXP (addr, 1);
+ addr = XEXP (addr, 0);
+ }
+ if (GET_CODE (offset) == VEC_DUPLICATE)
+ offset = XEXP (offset, 0);
+ if (CONST_VECTOR_P (offset))
+ offset = CONST_VECTOR_ELT (offset, 0);
+ if (!REG_P (addr) || !CONST_INT_P (offset))
+ {
+ addr = force_reg (<VnDI>mode, addr);
+ offset = const0_rtx;
+ }
+ emit_insn (gen_load<mode>_exec (operands[0], addr, offset,
+ GEN_INT (MEM_ADDR_SPACE (operands[1])),
+ operands[2], operands[3]));
+ DONE;
+ }
})
-(define_insn "mov<mode>_exec"
+(define_insn "*mov<mode>_1reg_exec"
[(set (match_operand:V_1REG 0 "nonimmediate_operand")
(vec_merge:V_1REG
(match_operand:V_1REG 1 "general_operand")
- (match_operand:V_1REG 2 "gcn_alu_or_unspec_operand")
- (match_operand:DI 3 "register_operand")))
+ (match_operand:V_1REG 2 "general_or_unspec_operand")
+ (match_operand:DI 3 "gcn_exec_operand")))
(clobber (match_scratch:<VnDI> 4))]
"!MEM_P (operands[0]) || REG_P (operands[1])"
- {@ [cons: =0, 1, 2, 3, =4; attrs: type, length]
- [v,vA,U0,e ,X ;vop1 ,4 ] v_mov_b32\t%0, %1
- [v,B ,U0,e ,X ;vop1 ,8 ] v_mov_b32\t%0, %1
- [v,v ,vA,cV,X ;vop2 ,4 ] v_cndmask_b32\t%0, %2, %1, vcc
- [v,vA,vA,Sv,X ;vop3a,8 ] v_cndmask_b32\t%0, %2, %1, %3
- [v,m ,U0,e ,&v;* ,16] #
- [m,v ,U0,e ,&v;* ,16] #
+ {@ [cons: =0, 1, 2, 3, =4; attrs: type, length, cdna, xnack]
+ [v ,vA,U0,e ,X ;vop1 ,4 ,* ,* ] v_mov_b32\t%0, %1
+ [v ,B ,U0,e ,X ;vop1 ,8 ,* ,* ] v_mov_b32\t%0, %1
+ [Rf ,v ,U0,e ,X ;flat ,12,* ,* ] flat_store%s0\t%A0, %1%O0
+ [Rf ,a ,U0,e ,X ;flat ,12,cdna2,* ] ^
+ [Rm ,v ,U0,e ,X ;flat ,12,* ,* ] global_store%s0\t%A0, %1%O0
+ [Rm ,a ,U0,e ,X ;flat ,12,cdna2,* ] ^
+ [Rl ,v ,U0,e ,X ;ds ,12,* ,* ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+ [v ,v ,vA,cV,X ;vop2 ,4 ,* ,* ] v_cndmask_b32\t%0, %2, %1, vcc
+ [v ,vA,vA,Sv,X ;vop3a,8 ,* ,* ] v_cndmask_b32\t%0, %2, %1, %3
+ [v ,RA,U0,e ,&v;* ,16,* ,* ] #
+ [RA ,v ,U0,e ,&v;* ,16,* ,* ] #
})
-(define_insn "*mov<mode>"
- [(set (match_operand:V_2REG 0 "nonimmediate_operand")
- (match_operand:V_2REG 1 "general_operand"))]
- ""
- {@ [cons: =0, 1; attrs: length, cdna]
- [v ,vDB;16,* ] v_mov_b32\t%L0, %L1\;v_mov_b32\t%H0, %H1
- [v ,a ;16,* ] v_accvgpr_read_b32\t%L0, %L1\;v_accvgpr_read_b32\t%H0, %H1
- [$a,v ;16,* ] v_accvgpr_write_b32\t%L0, %L1\;v_accvgpr_write_b32\t%H0, %H1
- [a ,a ;8 ,cdna2] v_accvgpr_mov_b32\t%L0, %L1\;v_accvgpr_mov_b32\t%H0, %H1
- }
- [(set_attr "type" "vmult,vmult,vmult,vmult")])
-
-(define_insn "mov<mode>_exec"
+(define_insn "*mov<mode>_2reg_exec"
[(set (match_operand:V_2REG 0 "nonimmediate_operand")
(vec_merge:V_2REG
(match_operand:V_2REG 1 "general_operand")
- (match_operand:V_2REG 2 "gcn_alu_or_unspec_operand")
- (match_operand:DI 3 "register_operand")))
+ (match_operand:V_2REG 2 "general_or_unspec_operand")
+ (match_operand:DI 3 "gcn_exec_operand")))
(clobber (match_scratch:<VnDI> 4))]
"!MEM_P (operands[0]) || REG_P (operands[1])"
- {@ [cons: =0, 1, 2, 3, =4; attrs: type, length]
- [v,vDB,U0 ,e ,X ;vmult,16] v_mov_b32\t%L0, %L1\;v_mov_b32\t%H0, %H1
- [v,v0 ,vDA0,cV,X ;vmult,16] v_cndmask_b32\t%L0, %L2, %L1, vcc\;v_cndmask_b32\t%H0, %H2, %H1, vcc
- [v,v0 ,vDA0,Sv,X ;vmult,16] v_cndmask_b32\t%L0, %L2, %L1, %3\;v_cndmask_b32\t%H0, %H2, %H1, %3
- [v,m ,U0 ,e ,&v;* ,16] #
- [m,v ,U0 ,e ,&v;* ,16] #
- })
-
-(define_insn "*mov<mode>_4reg"
- [(set (match_operand:V_4REG 0 "nonimmediate_operand")
- (match_operand:V_4REG 1 "general_operand"))]
- ""
- {@ [cons: =0, 1; attrs: type, length, cdna]
- [v ,vDB;vmult,16,* ] v_mov_b32\t%L0, %L1\; v_mov_b32\t%H0, %H1\; v_mov_b32\t%J0, %J1\; v_mov_b32\t%K0, %K1
- [v ,a ;vmult,32,* ] v_accvgpr_read_b32\t%L0, %L1\; v_accvgpr_read_b32\t%H0, %H1\; v_accvgpr_read_b32\t%J0, %J1\; v_accvgpr_read_b32\t%K0, %K1
- [$a,v ;vmult,32,* ] v_accvgpr_write_b32\t%L0, %L1\;v_accvgpr_write_b32\t%H0, %H1\;v_accvgpr_write_b32\t%J0, %J1\;v_accvgpr_write_b32\t%K0, %K1
- [a ,a ;vmult,32,cdna2] v_accvgpr_mov_b32\t%L0, %L1\; v_accvgpr_mov_b32\t%H0, %H1\; v_accvgpr_mov_b32\t%J0, %J1\; v_accvgpr_mov_b32\t%K0, %K1
+ {@ [cons: =0, 1, 2, 3, =4; attrs: type, length, cdna, xnack]
+ [v ,vDB,U0 ,e ,X ;vmult,16,* ,* ] v_mov_b32\t%L0, %L1\;v_mov_b32\t%H0, %H1
+ [v ,v0 ,vDA0,cV,X ;vmult,16,* ,* ] v_cndmask_b32\t%L0, %L2, %L1, vcc\;v_cndmask_b32\t%H0, %H2, %H1, vcc
+ [v ,v0 ,vDA0,Sv,X ;vmult,16,* ,* ] v_cndmask_b32\t%L0, %L2, %L1, %3\;v_cndmask_b32\t%H0, %H2, %H1, %3
+ [Rf ,v ,U0 ,e ,X ;flat ,12,* ,* ] flat_store_dwordx2\t%A0, %1%O0
+ [Rf ,a ,U0 ,e ,X ;flat ,12,cdna2,* ] ^
+ [Rm ,v ,U0 ,e ,X ;flat ,12,* ,* ] global_store_dwordx2\t%A0, %1%O0
+ [Rm ,a ,U0 ,e ,X ;flat ,12,cdna2,* ] ^
+ [Rl ,v ,U0 ,e ,X ;ds ,12,* ,* ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+ [v ,RA ,U0 ,e ,&v;* ,16,* ,* ] #
+ [RA ,v ,U0 ,e ,&v;* ,16,* ,* ] #
})
-(define_insn "mov<mode>_exec"
+(define_insn "*mov<mode>_4reg_exec"
[(set (match_operand:V_4REG 0 "nonimmediate_operand")
(vec_merge:V_4REG
(match_operand:V_4REG 1 "general_operand")
- (match_operand:V_4REG 2 "gcn_alu_or_unspec_operand")
- (match_operand:DI 3 "register_operand")))
+ (match_operand:V_4REG 2 "general_or_unspec_operand")
+ (match_operand:DI 3 "gcn_exec_operand")))
(clobber (match_scratch:<VnDI> 4))]
"!MEM_P (operands[0]) || REG_P (operands[1])"
- {@ [cons: =0, 1, 2, 3, =4; attrs: type, length]
- [v,vDB,U0 ,e ,X ;vmult,32] v_mov_b32\t%L0, %L1\;v_mov_b32\t%H0, %H1\;v_mov_b32\t%J0, %J1\;v_mov_b32\t%K0, %K1
- [v,v0 ,vDA0,cV,X ;vmult,32] v_cndmask_b32\t%L0, %L2, %L1, vcc\;v_cndmask_b32\t%H0, %H2, %H1, vcc\;v_cndmask_b32\t%J0, %J2, %J1, vcc\;v_cndmask_b32\t%K0, %K2, %K1, vcc
- [v,v0 ,vDA0,Sv,X ;vmult,32] v_cndmask_b32\t%L0, %L2, %L1, %3\;v_cndmask_b32\t%H0, %H2, %H1, %3\;v_cndmask_b32\t%J0, %J2, %J1, %3\;v_cndmask_b32\t%K0, %K2, %K1, %3
- [v,m ,U0 ,e ,&v;* ,32] #
- [m,v ,U0 ,e ,&v;* ,32] #
+ {@ [cons: =0, 1, 2, 3, =4; attrs: type, length, cdna, xnack]
+ [v ,vDB,U0 ,e ,X ;vmult,32,* ,* ] v_mov_b32\t%L0, %L1\;v_mov_b32\t%H0, %H1\;v_mov_b32\t%J0, %J1\;v_mov_b32\t%K0, %K1
+ [v ,v0 ,vDA0,cV,X ;vmult,32,* ,* ] v_cndmask_b32\t%L0, %L2, %L1, vcc\;v_cndmask_b32\t%H0, %H2, %H1, vcc\;v_cndmask_b32\t%J0, %J2, %J1, vcc\;v_cndmask_b32\t%K0, %K2, %K1, vcc
+ [v ,v0 ,vDA0,Sv,X ;vmult,32,* ,* ] v_cndmask_b32\t%L0, %L2, %L1, %3\;v_cndmask_b32\t%H0, %H2, %H1, %3\;v_cndmask_b32\t%J0, %J2, %J1, %3\;v_cndmask_b32\t%K0, %K2, %K1, %3
+ [Rf ,v ,U0 ,e ,X ;flat ,12,* ,* ] flat_store_dwordx4\t%A0, %1%O0
+ [Rf ,a ,U0 ,e ,X ;flat ,12,cdna2,* ] ^
+ [Rm ,v ,U0 ,e ,X ;flat ,12,* ,* ] global_store_dwordx4\t%A0, %1%O0
+ [Rm ,a ,U0 ,e ,X ;flat ,12,cdna2,* ] ^
+ [v ,RA ,U0 ,e ,&v;* ,32,* ,* ] #
+ [RA ,v ,U0 ,e ,&v;* ,32,* ,* ] #
+ })
+
+;; A mov<mode>_exec insn for partial loads that doesn't use an explicit MEM
+;; and therefore can't get converted to an unmasked load by reload.
+;; Other architectures don't seem to suffer from this problem, but that
+;; may be because they don't have early-clobber constraints to cause reloads.
+(define_insn "load<mode>_exec"
+ [(set (match_operand:V_MOV 0 "register_operand")
+ (vec_merge:V_MOV
+ (unspec:V_MOV [
+ (plus:<VnDI> (match_operand:<VnDI> 1 "register_operand")
+ (match_operand:DI 2 "immediate_operand"))
+ (match_operand:SI 3 "immediate_operand")
+ (mem:BLK (const_int 0))]
+ UNSPEC_MASKLOAD)
+ (match_operand:V_MOV 4 "general_or_unspec_operand")
+ (match_operand:DI 5 "gcn_exec_operand")))]
+ ""
+ {@ [cons: =0, 1, 2, 3, 4, 5; attrs: type, length, cdna, xnack]
+ [v ,v,i,Pf,U0,e ;flat ,12,* ,off] flat_load%o0\t%0, %1 offset:%2\;s_waitcnt\t0
+ [&v ,v,i,Pf,U0,e ;flat ,12,* ,on ] ^
+ [^a ,v,i,Pf,U0,e ;flat ,12,cdna2,off] ^
+ [&^a,v,i,Pf,U0,e ;flat ,12,cdna2,on ] ^
+ [v ,v,i,Pm,U0,e ;flat ,12,* ,off] global_load%o0\t%0, %1, off offset:%2\;s_waitcnt\tvmcnt(0)
+ [&v ,v,i,Pm,U0,e ;flat ,12,* ,on ] ^
+ [^a ,v,i,Pm,U0,e ;flat ,12,cdna2,off] ^
+ [&^a,v,i,Pm,U0,e ;flat ,12,cdna2,on ] ^
+ [v ,v,i,Pl,U0,e ;ds ,12,* ,* ] ds_read%b0\t%0, %1 offset:%2\;s_waitcnt\tlgkmcnt(0)
})
; A SGPR-base load looks like:
@@ -588,7 +686,7 @@ (define_insn "@mov<mode>_sgprbase"
[m,v ,&v;* ,12] #
})
-; Expand scalar addresses into gather/scatter patterns
+; Expand scalar base addresses into vectors of addresses
(define_split
[(set (match_operand:V_MOV 0 "memory_operand")
@@ -596,16 +694,12 @@ (define_split
[(match_operand:V_MOV 1 "general_operand")]
UNSPEC_SGPRBASE))
(clobber (match_scratch:<VnDI> 2))]
- ""
- [(set (mem:BLK (scratch))
- (unspec:BLK [(match_dup 5) (match_dup 1) (match_dup 6) (match_dup 7)]
- UNSPEC_SCATTER))]
+ "!VECTOR_MODE_P (GET_MODE (XEXP (operands[0], 0)))"
+ [(set (match_dup 0) (match_dup 1))]
{
- operands[5] = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
+ operands[0] = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
operands[0],
operands[2]);
- operands[6] = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[0]));
- operands[7] = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[0]));
})
(define_split
@@ -615,18 +709,17 @@ (define_split
(match_operand:V_MOV 2 "")
(match_operand:DI 3 "gcn_exec_reg_operand")))
(clobber (match_scratch:<VnDI> 4))]
- ""
- [(set (mem:BLK (scratch))
- (unspec:BLK [(match_dup 5) (match_dup 1)
- (match_dup 6) (match_dup 7) (match_dup 3)]
- UNSPEC_SCATTER))]
+ "!VECTOR_MODE_P (GET_MODE (XEXP (operands[0], 0)))"
+ [(parallel [(set (match_dup 0)
+ (vec_merge:V_MOV (match_dup 1) (match_dup 2) (match_dup 3)))
+ (clobber (match_dup 4))])]
{
- operands[5] = gcn_expand_scalar_to_vector_address (<MODE>mode,
- operands[3],
- operands[0],
- operands[4]);
- operands[6] = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[0]));
- operands[7] = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[0]));
+ rtx mem = gcn_expand_scalar_to_vector_address (<MODE>mode, operands[3],
+ operands[0], operands[4]);
+ if (rtx_equal_p (operands[0], operands[2]))
+ operands[0] = operands[2] = mem;
+ else
+ operands[0] = mem;
})
(define_split
@@ -635,17 +728,12 @@ (define_split
[(match_operand:V_MOV 1 "memory_operand")]
UNSPEC_SGPRBASE))
(clobber (match_scratch:<VnDI> 2))]
- ""
- [(set (match_dup 0)
- (unspec:V_MOV [(match_dup 5) (match_dup 6) (match_dup 7)
- (mem:BLK (scratch))]
- UNSPEC_GATHER))]
+ "!VECTOR_MODE_P (GET_MODE (XEXP (operands[1], 0)))"
+ [(set (match_dup 0) (match_dup 1))]
{
- operands[5] = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
+ operands[1] = gcn_expand_scalar_to_vector_address (<MODE>mode, NULL,
operands[1],
operands[2]);
- operands[6] = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[1]));
- operands[7] = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[1]));
})
(define_split
@@ -655,21 +743,14 @@ (define_split
(match_operand:V_MOV 2 "")
(match_operand:DI 3 "gcn_exec_reg_operand")))
(clobber (match_scratch:<VnDI> 4))]
- ""
- [(set (match_dup 0)
- (vec_merge:V_MOV
- (unspec:V_MOV [(match_dup 5) (match_dup 6) (match_dup 7)
- (mem:BLK (scratch))]
- UNSPEC_GATHER)
- (match_dup 2)
- (match_dup 3)))]
+ "!VECTOR_MODE_P (GET_MODE (XEXP (operands[1], 0)))"
+ [(const_int 0)]
{
- operands[5] = gcn_expand_scalar_to_vector_address (<MODE>mode,
- operands[3],
- operands[1],
- operands[4]);
- operands[6] = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[1]));
- operands[7] = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[1]));
+ rtx mem = gcn_expand_scalar_to_vector_address (<MODE>mode, operands[3],
+ operands[1], operands[4]);
+ emit_insn (gen_mov<mode>_exec (operands[0], mem, operands[2],
+ operands[3]));
+ DONE;
})
; TODO: Add zero/sign extending variants.
@@ -976,38 +1057,6 @@ (define_expand "vec_init<V_MOV:mode><V_MOV_ALT:mode>"
;; }}}
;; {{{ Scatter / Gather
-;; GCN does not have an instruction for loading a vector from contiguous
-;; memory so *all* loads and stores are eventually converted to scatter
-;; or gather.
-;;
-;; GCC does not permit MEM to hold vectors of addresses, so we must use an
-;; unspec. The unspec formats are as follows:
-;;
-;; (unspec:V??
-;; [(<address expression>)
-;; (<addr_space_t>)
-;; (<use_glc>)
-;; (mem:BLK (scratch))]
-;; UNSPEC_GATHER)
-;;
-;; (unspec:BLK
-;; [(<address expression>)
-;; (<source register>)
-;; (<addr_space_t>)
-;; (<use_glc>)
-;; (<exec>)]
-;; UNSPEC_SCATTER)
-;;
-;; - Loads are expected to be wrapped in a vec_merge, so do not need <exec>.
-;; - The mem:BLK does not contain any real information, but indicates that an
-;; unknown memory read is taking place. Stores are expected to use a similar
-;; mem:BLK outside the unspec.
-;; - The address space and glc (volatile) fields are there to replace the
-;; fields normally found in a MEM.
-;; - Multiple forms of address expression are supported, below.
-;;
-;; TODO: implement combined gather and zero_extend, but only for -msram-ecc=on
-
(define_expand "gather_load<mode><vnsi>"
[(match_operand:V_MOV 0 "register_operand")
(match_operand:DI 1 "register_operand")
@@ -1016,17 +1065,10 @@ (define_expand "gather_load<mode><vnsi>"
(match_operand:SI 4 "gcn_alu_operand")]
""
{
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
- operands[2], operands[4],
- INTVAL (operands[3]), NULL);
-
- if (GET_MODE (addr) == <VnDI>mode)
- emit_insn (gen_gather<mode>_insn_1offset (operands[0], addr, const0_rtx,
- const0_rtx, const0_rtx));
- else
- emit_insn (gen_gather<mode>_insn_2offsets (operands[0], operands[1],
- addr, const0_rtx, const0_rtx,
- const0_rtx));
+ emit_move_insn (operands[0],
+ gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[1], operands[2], operands[4],
+ INTVAL (operands[3]), false, NULL));
DONE;
})
@@ -1038,121 +1080,13 @@ (define_expand "gather_load<mode><vndi>"
(match_operand:SI 4 "gcn_alu_operand")]
""
{
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
- operands[2], operands[4],
- INTVAL (operands[3]), NULL);
-
- emit_insn (gen_gather<mode>_insn_1offset (operands[0], addr, const0_rtx,
- const0_rtx, const0_rtx));
+ emit_move_insn (operands[0],
+ gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[1], operands[2], operands[4],
+ INTVAL (operands[3]), false, NULL));
DONE;
})
-; Allow any address expression
-(define_expand "gather<mode>_expr<exec>"
- [(set (match_operand:V_MOV 0 "register_operand")
- (unspec:V_MOV
- [(match_operand 1 "")
- (match_operand 2 "immediate_operand")
- (match_operand 3 "immediate_operand")
- (mem:BLK (scratch))]
- UNSPEC_GATHER))]
- ""
- {})
-
-(define_insn "gather<mode>_insn_1offset<exec>"
- [(set (match_operand:V_MOV 0 "register_operand" "=v,a,&v,&a")
- (unspec:V_MOV
- [(plus:<VnDI> (match_operand:<VnDI> 1 "register_operand" " v,v, v, v")
- (vec_duplicate:<VnDI>
- (match_operand 2 "immediate_operand" " n,n, n, n")))
- (match_operand 3 "immediate_operand" " n,n, n, n")
- (match_operand 4 "immediate_operand" " n,n, n, n")
- (mem:BLK (scratch))]
- UNSPEC_GATHER))]
- "(AS_FLAT_P (INTVAL (operands[3]))
- && ((unsigned HOST_WIDE_INT)INTVAL(operands[2]) < 0x1000))
- || (AS_GLOBAL_P (INTVAL (operands[3]))
- && (((unsigned HOST_WIDE_INT)INTVAL(operands[2]) + 0x1000) < 0x2000))"
- {
- addr_space_t as = INTVAL (operands[3]);
- const char *glc = INTVAL (operands[4]) ? TARGET_GLC_NAME : "";
-
- static char buf[200];
- if (AS_FLAT_P (as))
- sprintf (buf, "flat_load%%o0\t%%0, %%1 offset:%%2%s\;s_waitcnt\t0", glc);
- else if (AS_GLOBAL_P (as))
- sprintf (buf, "global_load%%o0\t%%0, %%1, off offset:%%2%s\;"
- "s_waitcnt\tvmcnt(0)", glc);
- else
- gcc_unreachable ();
-
- return buf;
- }
- [(set_attr "type" "flat")
- (set_attr "flatmemaccess" "load")
- (set_attr "length" "12")
- (set_attr "cdna" "*,cdna2,*,cdna2")
- (set_attr "xnack" "off,off,on,on")])
-
-(define_insn "gather<mode>_insn_1offset_ds<exec>"
- [(set (match_operand:V_MOV 0 "register_operand" "=v,a")
- (unspec:V_MOV
- [(plus:<VnSI> (match_operand:<VnSI> 1 "register_operand" " v,v")
- (vec_duplicate:<VnSI>
- (match_operand 2 "immediate_operand" " n,n")))
- (match_operand 3 "immediate_operand" " n,n")
- (match_operand 4 "immediate_operand" " n,n")
- (mem:BLK (scratch))]
- UNSPEC_GATHER))]
- "(AS_ANY_DS_P (INTVAL (operands[3]))
- && ((unsigned HOST_WIDE_INT)INTVAL(operands[2]) < 0x10000))"
- {
- addr_space_t as = INTVAL (operands[3]);
- static char buf[200];
- sprintf (buf, "ds_read%%b0\t%%0, %%1 offset:%%2%s\;s_waitcnt\tlgkmcnt(0)",
- (AS_GDS_P (as) ? " gds" : ""));
- return buf;
- }
- [(set_attr "type" "ds")
- (set_attr "length" "12")
- (set_attr "cdna" "*,cdna2")])
-
-(define_insn "gather<mode>_insn_2offsets<exec>"
- [(set (match_operand:V_MOV 0 "register_operand" "=v,a,&v,&a")
- (unspec:V_MOV
- [(plus:<VnDI>
- (plus:<VnDI>
- (vec_duplicate:<VnDI>
- (match_operand:DI 1 "register_operand" "Sv,Sv,Sv,Sv"))
- (sign_extend:<VnDI>
- (match_operand:<VnSI> 2 "register_operand" " v, v, v, v")))
- (vec_duplicate:<VnDI> (match_operand 3 "immediate_operand"
- " n, n, n, n")))
- (match_operand 4 "immediate_operand" " n, n, n, n")
- (match_operand 5 "immediate_operand" " n, n, n, n")
- (mem:BLK (scratch))]
- UNSPEC_GATHER))]
- "(AS_GLOBAL_P (INTVAL (operands[4]))
- && (((unsigned HOST_WIDE_INT)INTVAL(operands[3]) + 0x1000) < 0x2000))"
- {
- addr_space_t as = INTVAL (operands[4]);
- const char *glc = INTVAL (operands[5]) ? TARGET_GLC_NAME : "";
-
- static char buf[200];
- if (AS_GLOBAL_P (as))
- sprintf (buf, "global_load%%o0\t%%0, %%2, %%1 offset:%%3%s\;"
- "s_waitcnt\tvmcnt(0)", glc);
- else
- gcc_unreachable ();
-
- return buf;
- }
- [(set_attr "type" "flat")
- (set_attr "flatmemaccess" "load")
- (set_attr "length" "12")
- (set_attr "cdna" "*,cdna2,*,cdna2")
- (set_attr "xnack" "off,off,on,on")])
-
(define_expand "scatter_store<mode><vnsi>"
[(match_operand:DI 0 "register_operand")
(match_operand:<VnSI> 1 "register_operand")
@@ -1161,17 +1095,10 @@ (define_expand "scatter_store<mode><vnsi>"
(match_operand:V_MOV 4 "register_operand")]
""
{
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
- operands[1], operands[3],
- INTVAL (operands[2]), NULL);
-
- if (GET_MODE (addr) == <VnDI>mode)
- emit_insn (gen_scatter<mode>_insn_1offset (addr, const0_rtx, operands[4],
- const0_rtx, const0_rtx));
- else
- emit_insn (gen_scatter<mode>_insn_2offsets (operands[0], addr,
- const0_rtx, operands[4],
- const0_rtx, const0_rtx));
+ emit_move_insn (gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[0], operands[1], operands[3],
+ INTVAL (operands[2]), false, NULL),
+ operands[4]);
DONE;
})
@@ -1183,117 +1110,13 @@ (define_expand "scatter_store<mode><vndi>"
(match_operand:V_MOV 4 "register_operand")]
""
{
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
- operands[1], operands[3],
- INTVAL (operands[2]), NULL);
-
- emit_insn (gen_scatter<mode>_insn_1offset (addr, const0_rtx, operands[4],
- const0_rtx, const0_rtx));
+ emit_move_insn (gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[0], operands[1], operands[3],
+ INTVAL (operands[2]), false, NULL),
+ operands[4]);
DONE;
})
-; Allow any address expression
-(define_expand "scatter<mode>_expr<exec_scatter>"
- [(set (mem:BLK (scratch))
- (unspec:BLK
- [(match_operand:<VnDI> 0 "")
- (match_operand:V_MOV 1 "register_operand")
- (match_operand 2 "immediate_operand")
- (match_operand 3 "immediate_operand")]
- UNSPEC_SCATTER))]
- ""
- {})
-
-(define_insn "scatter<mode>_insn_1offset<exec_scatter>"
- [(set (mem:BLK (scratch))
- (unspec:BLK
- [(plus:<VnDI> (match_operand:<VnDI> 0 "register_operand" "v,v")
- (vec_duplicate:<VnDI>
- (match_operand 1 "immediate_operand" "n,n")))
- (match_operand:V_MOV 2 "register_operand" "v,a")
- (match_operand 3 "immediate_operand" "n,n")
- (match_operand 4 "immediate_operand" "n,n")]
- UNSPEC_SCATTER))]
- "(AS_FLAT_P (INTVAL (operands[3]))
- && (INTVAL(operands[1]) == 0
- || ((unsigned HOST_WIDE_INT)INTVAL(operands[1]) < 0x1000)))
- || (AS_GLOBAL_P (INTVAL (operands[3]))
- && (((unsigned HOST_WIDE_INT)INTVAL(operands[1]) + 0x1000) < 0x2000))"
- {
- addr_space_t as = INTVAL (operands[3]);
- const char *glc = INTVAL (operands[4]) ? TARGET_GLC_NAME : "";
-
- static char buf[200];
- if (AS_FLAT_P (as))
- sprintf (buf, "flat_store%%s2\t%%0, %%2 offset:%%1%s", glc);
- else if (AS_GLOBAL_P (as))
- sprintf (buf, "global_store%%s2\t%%0, %%2, off offset:%%1%s", glc);
- else
- gcc_unreachable ();
-
- return buf;
- }
- [(set_attr "type" "flat")
- (set_attr "flatmemaccess" "store")
- (set_attr "length" "12")
- (set_attr "cdna" "*,cdna2")])
-
-(define_insn "scatter<mode>_insn_1offset_ds<exec_scatter>"
- [(set (mem:BLK (scratch))
- (unspec:BLK
- [(plus:<VnSI> (match_operand:<VnSI> 0 "register_operand" "v,v")
- (vec_duplicate:<VnSI>
- (match_operand 1 "immediate_operand" "n,n")))
- (match_operand:V_MOV 2 "register_operand" "v,a")
- (match_operand 3 "immediate_operand" "n,n")
- (match_operand 4 "immediate_operand" "n,n")]
- UNSPEC_SCATTER))]
- "(AS_ANY_DS_P (INTVAL (operands[3]))
- && ((unsigned HOST_WIDE_INT)INTVAL(operands[1]) < 0x10000))"
- {
- addr_space_t as = INTVAL (operands[3]);
- static char buf[200];
- sprintf (buf, "ds_write%%b2\t%%0, %%2 offset:%%1%s\;s_waitcnt\tlgkmcnt(0)",
- (AS_GDS_P (as) ? " gds" : ""));
- return buf;
- }
- [(set_attr "type" "ds")
- (set_attr "length" "12")
- (set_attr "cdna" "*,cdna2")])
-
-(define_insn "scatter<mode>_insn_2offsets<exec_scatter>"
- [(set (mem:BLK (scratch))
- (unspec:BLK
- [(plus:<VnDI>
- (plus:<VnDI>
- (vec_duplicate:<VnDI>
- (match_operand:DI 0 "register_operand" "Sv,Sv"))
- (sign_extend:<VnDI>
- (match_operand:<VnSI> 1 "register_operand" "v,v")))
- (vec_duplicate:<VnDI> (match_operand 2 "immediate_operand" "n,n")))
- (match_operand:V_MOV 3 "register_operand" "v,a")
- (match_operand 4 "immediate_operand" "n,n")
- (match_operand 5 "immediate_operand" "n,n")]
- UNSPEC_SCATTER))]
- "(AS_GLOBAL_P (INTVAL (operands[4]))
- && (((unsigned HOST_WIDE_INT)INTVAL(operands[2]) + 0x1000) < 0x2000))"
- {
- addr_space_t as = INTVAL (operands[4]);
- const char *glc = INTVAL (operands[5]) ? TARGET_GLC_NAME : "";
-
- static char buf[200];
- if (AS_GLOBAL_P (as))
- sprintf (buf, "global_store%%s3\t%%1, %%3, %%0 offset:%%2%s", glc);
- else
- gcc_unreachable ();
-
- return buf;
- }
- [(set_attr "type" "flat")
- (set_attr "flatmemaccess" "store")
- (set_attr "length" "12")
- (set_attr "cdna" "*,cdna2")])
-
;; }}}
;; {{{ Permutations
@@ -4120,13 +3943,10 @@ (define_expand "maskload<mode>di"
""
{
rtx exec = force_reg (DImode, operands[2]);
- rtx addr = gcn_expand_scalar_to_vector_address
+ rtx mem = gcn_expand_scalar_to_vector_address
(<MODE>mode, exec, operands[1], gen_rtx_SCRATCH (<VnDI>mode));
- rtx as = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[1]));
- rtx v = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[1]));
-
- emit_insn (gen_gather<mode>_expr_exec (operands[0], addr, as, v,
- gcn_gen_undef (<MODE>mode), exec));
+ emit_insn (gen_mov<mode>_exec (operands[0], mem,
+ gcn_gen_undef (<MODE>mode), exec));
DONE;
})
@@ -4137,11 +3957,10 @@ (define_expand "maskstore<mode>di"
""
{
rtx exec = force_reg (DImode, operands[2]);
- rtx addr = gcn_expand_scalar_to_vector_address
+ rtx mem = gcn_expand_scalar_to_vector_address
(<MODE>mode, exec, operands[0], gen_rtx_SCRATCH (<VnDI>mode));
- rtx as = gen_rtx_CONST_INT (VOIDmode, MEM_ADDR_SPACE (operands[0]));
- rtx v = gen_rtx_CONST_INT (VOIDmode, MEM_VOLATILE_P (operands[0]));
- emit_insn (gen_scatter<mode>_expr_exec (addr, operands[1], as, v, exec));
+ emit_insn (gen_mov<mode>_exec (mem, operands[1],
+ gcn_gen_undef (<MODE>mode), exec));
DONE;
})
@@ -4156,51 +3975,30 @@ (define_expand "mask_gather_load<mode><vnsi>"
""
{
rtx exec = force_reg (DImode, operands[5]);
-
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
- operands[2], operands[4],
- INTVAL (operands[3]), exec);
-
- if (GET_MODE (addr) == <VnDI>mode)
- emit_insn (gen_gather<mode>_insn_1offset_exec (operands[0], addr,
- const0_rtx, const0_rtx,
- const0_rtx,
- gcn_gen_undef
- (<MODE>mode),
- exec));
- else
- emit_insn (gen_gather<mode>_insn_2offsets_exec (operands[0], operands[1],
- addr, const0_rtx,
- const0_rtx, const0_rtx,
- gcn_gen_undef
- (<MODE>mode),
- exec));
+ rtx srcmem = gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[1], operands[2], operands[4],
+ INTVAL (operands[3]), false, exec);
+ emit_insn (gen_mov<mode>_exec (operands[0], srcmem,
+ gcn_gen_undef (<MODE>mode), exec));
DONE;
})
(define_expand "mask_gather_load<mode><vndi>"
- [(set:V_MOV (match_operand:V_MOV 0 "register_operand")
- (unspec:V_MOV
- [(match_operand:DI 1 "register_operand")
- (match_operand:<VnDI> 2 "register_operand")
- (match_operand 3 "immediate_operand")
- (match_operand:SI 4 "gcn_alu_operand")
- (match_operand:DI 5 "")
- (match_operand:V_MOV 6 "maskload_else_operand")]
- UNSPEC_GATHER))]
+ [(match_operand:V_MOV 0 "register_operand")
+ (match_operand:DI 1 "register_operand")
+ (match_operand:<VnDI> 2 "register_operand")
+ (match_operand 3 "immediate_operand")
+ (match_operand:SI 4 "gcn_alu_operand")
+ (match_operand:DI 5 "")
+ (match_operand:V_MOV 6 "maskload_else_operand")]
""
{
rtx exec = force_reg (DImode, operands[5]);
-
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
- operands[2], operands[4],
- INTVAL (operands[3]), exec);
-
- emit_insn (gen_gather<mode>_insn_1offset_exec (operands[0], addr,
- const0_rtx, const0_rtx,
- const0_rtx,
- gcn_gen_undef (<MODE>mode),
- exec));
+ rtx srcmem = gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[1], operands[2], operands[4],
+ INTVAL (operands[3]), false, exec);
+ emit_insn (gen_mov<mode>_exec (operands[0], srcmem,
+ gcn_gen_undef (<MODE>mode), exec));
DONE;
})
@@ -4214,21 +4012,11 @@ (define_expand "mask_scatter_store<mode><vnsi>"
""
{
rtx exec = force_reg (DImode, operands[5]);
-
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
- operands[1], operands[3],
- INTVAL (operands[2]), exec);
-
- if (GET_MODE (addr) == <VnDI>mode)
- emit_insn (gen_scatter<mode>_insn_1offset_exec (addr, const0_rtx,
- operands[4], const0_rtx,
- const0_rtx,
- exec));
- else
- emit_insn (gen_scatter<mode>_insn_2offsets_exec (operands[0], addr,
- const0_rtx, operands[4],
- const0_rtx, const0_rtx,
- exec));
+ rtx destmem = gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[0], operands[1], operands[3],
+ INTVAL (operands[2]), false, exec);
+ emit_insn (gen_mov<mode>_exec (destmem, operands[4],
+ gcn_gen_undef (<MODE>mode), exec));
DONE;
})
@@ -4242,14 +4030,11 @@ (define_expand "mask_scatter_store<mode><vndi>"
""
{
rtx exec = force_reg (DImode, operands[5]);
-
- rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
- operands[1], operands[3],
- INTVAL (operands[2]), exec);
-
- emit_insn (gen_scatter<mode>_insn_1offset_exec (addr, const0_rtx,
- operands[4], const0_rtx,
- const0_rtx, exec));
+ rtx destmem = gcn_gen_vector_mem (<MODE>mode, DEFAULT_ADDR_SPACE,
+ operands[0], operands[1], operands[3],
+ INTVAL (operands[2]), false, exec);
+ emit_insn (gen_mov<mode>_exec (destmem, operands[4],
+ gcn_gen_undef (<MODE>mode), exec));
DONE;
})
@@ -1431,8 +1431,6 @@ GEN_VN (addc,si3, A(rtx dest, rtx src1, rtx src2, rtx vccout, rtx vccin),
GEN_VN (and,si3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
GEN_VNM_NOEXEC (ds_bpermute,, A(rtx dest, rtx addr, rtx src, rtx exec),
A(dest, addr, src, exec))
-GEN_VNM (gather,_expr, A(rtx dest, rtx addr, rtx as, rtx vol),
- A(dest, addr, as, vol))
GEN_VN (sub,si3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
GEN_VN_NOEXEC (vec_series,si, A(rtx dest, rtx x, rtx c), A(dest, x, c))
@@ -1471,11 +1469,11 @@ gcn_stepped_zero_int_parallel_p (rtx op, int step)
/* {{{ Addresses, pointers and moves. */
/* Return true is REG is a valid place to store a pointer,
- for instructions that require an SGPR.
- FIXME rename. */
+ for instructions that require an SGPR. Also check that the address
+ width is correct for the address space. */
static bool
-gcn_address_register_p (rtx reg, machine_mode mode, bool strict)
+gcn_scalar_address_register_p (rtx reg, machine_mode as_mode, bool strict)
{
if (GET_CODE (reg) == SUBREG)
reg = SUBREG_REG (reg);
@@ -1483,7 +1481,7 @@ gcn_address_register_p (rtx reg, machine_mode mode, bool strict)
if (!REG_P (reg))
return false;
- if (GET_MODE (reg) != mode)
+ if (GET_MODE (reg) != as_mode)
return false;
int regno = REGNO (reg);
@@ -1504,10 +1502,11 @@ gcn_address_register_p (rtx reg, machine_mode mode, bool strict)
}
/* Return true is REG is a valid place to store a pointer,
- for instructions that require a VGPR. */
+ for instructions that require a VGPR. Also check that the address
+ width is correct for the address space. */
static bool
-gcn_vec_address_register_p (rtx reg, machine_mode mode, bool strict)
+gcn_vec_address_register_p (rtx reg, machine_mode as_mode, bool strict)
{
if (GET_CODE (reg) == SUBREG)
reg = SUBREG_REG (reg);
@@ -1515,7 +1514,12 @@ gcn_vec_address_register_p (rtx reg, machine_mode mode, bool strict)
if (!REG_P (reg))
return false;
- if (GET_MODE (reg) != mode)
+ /* Vector addresses are allowed, but MODE is always specified scalar
+ (otherwise the number of lanes would have to match too). */
+ machine_mode scalar_reg_mode = (VECTOR_MODE_P (GET_MODE (reg))
+ ? GET_MODE_INNER (GET_MODE (reg))
+ : GET_MODE (reg));
+ if (scalar_reg_mode != as_mode)
return false;
int regno = REGNO (reg);
@@ -1534,25 +1538,64 @@ gcn_vec_address_register_p (rtx reg, machine_mode mode, bool strict)
return VGPR_REGNO_P (regno);
}
+/* Return true if REG is a valid place to store a pointer,
+ for instructions that require an SGPR or VGPR, depending on the
+ mode of the address and the data type:
+
+ Scalar load, scalar address: VGPR.
+ Vector load, scalar base: SGPR. (These get expanded later.)
+ Vector load, vector of addresses: VGPR.
+
+ This function is only suitable for FLAT and GLOBAL address spaces. */
+
+static bool
+gcn_auto_address_register_p (rtx addr, machine_mode data_mode, bool strict)
+{
+ machine_mode addr_mode = GET_MODE (addr);
+
+ if (!VECTOR_MODE_P (data_mode))
+ return (!VECTOR_MODE_P (addr_mode)
+ && gcn_vec_address_register_p (addr, DImode, strict));
+ else if (!VECTOR_MODE_P (addr_mode))
+ return gcn_scalar_address_register_p (addr, DImode, strict);
+ else
+ return gcn_vec_address_register_p (addr, DImode, strict);
+}
+
/* Return true if X would be valid inside a MEM using the Flat address
space. */
bool
-gcn_flat_address_p (rtx x, machine_mode mode)
+gcn_flat_address_p (rtx x, machine_mode data_mode, bool strict)
{
- bool vec_mode = (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
- || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT);
-
- if (vec_mode && gcn_address_register_p (x, DImode, false))
- return true;
-
- if (!vec_mode && gcn_vec_address_register_p (x, DImode, false))
+ if (gcn_auto_address_register_p (x, data_mode, strict))
return true;
if (GET_CODE (x) == PLUS
- && gcn_vec_address_register_p (XEXP (x, 0), DImode, false)
- && CONST_INT_P (XEXP (x, 1)))
- return true;
+ && gcn_auto_address_register_p (XEXP (x, 0), data_mode, strict))
+ {
+ rtx x1 = XEXP (x, 1);
+
+ if (GET_CODE (x1) == VEC_DUPLICATE
+ && VECTOR_MODE_P (GET_MODE (x1))
+ && GET_MODE_INNER (GET_MODE (x1)) == DImode)
+ x1 = XEXP (x1, 0);
+ else if (GET_CODE (x1) == CONST_VECTOR
+ && VECTOR_MODE_P (GET_MODE (x1))
+ && GET_MODE_INNER (GET_MODE (x1)) == DImode
+ && gcn_constant_p (x1))
+ x1 = XVECEXP (x1, 0, 0);
+
+ if (GET_CODE (x1) == CONST_INT)
+ {
+ int offsetbits = (TARGET_11BIT_GLOBAL_OFFSET ? 11 : 12);
+ if (INTVAL (x1) >= 0 && INTVAL (x1) < (1 << offsetbits)
+ /* The low bits of the offset are ignored, even when
+ they're meant to realign the pointer. */
+ && !(INTVAL (x1) & 0x3))
+ return true;
+ }
+ }
return false;
}
@@ -1561,15 +1604,34 @@ gcn_flat_address_p (rtx x, machine_mode mode)
address space. */
bool
-gcn_scalar_flat_address_p (rtx x)
+gcn_scalar_flat_address_p (rtx x, bool strict)
{
- if (gcn_address_register_p (x, DImode, false))
+ if (gcn_scalar_address_register_p (x, DImode, strict))
return true;
if (GET_CODE (x) == PLUS
- && gcn_address_register_p (XEXP (x, 0), DImode, false)
- && CONST_INT_P (XEXP (x, 1)))
- return true;
+ && gcn_scalar_address_register_p (XEXP (x, 0), DImode, strict))
+ {
+ rtx x1 = XEXP (x, 1);
+
+ /* FIXME: This is disabled because of the mode mismatch between
+ SImode (for the address or m0 register) and the DImode PLUS.
+ We'll need a zero_extend or similar.
+
+ if (gcn_m0_register_p (x1, SImode, strict)
+ || gcn_scalar_address_register_p (x1, SImode, strict))
+ return true;
+ else*/
+
+ if (GET_CODE (x1) == CONST_INT)
+ {
+ if (INTVAL (x1) >= 0 && INTVAL (x1) < (1 << 20)
+ /* The low bits of the offset are ignored, even when
+ they're meant to realign the pointer. */
+ && !(INTVAL (x1) & 0x3))
+ return true;
+ }
+ }
return false;
}
@@ -1592,15 +1654,35 @@ gcn_scalar_flat_mem_p (rtx x)
address spaces. */
bool
-gcn_ds_address_p (rtx x)
+gcn_ds_address_p (rtx x, bool strict = false)
{
- if (gcn_vec_address_register_p (x, SImode, false))
+ if (gcn_vec_address_register_p (x, SImode, strict))
return true;
if (GET_CODE (x) == PLUS
- && gcn_vec_address_register_p (XEXP (x, 0), SImode, false)
- && CONST_INT_P (XEXP (x, 1)))
- return true;
+ && gcn_vec_address_register_p (XEXP (x, 0), SImode, strict))
+ {
+ rtx x0 = XEXP (x, 0);
+ rtx x1 = XEXP (x, 1);
+ if (!gcn_vec_address_register_p (x0, DImode, strict))
+ return false;
+ if (GET_CODE (x1) == REG)
+ {
+ if (GET_CODE (x1) != REG
+ || (REGNO (x1) <= FIRST_PSEUDO_REGISTER
+ && !gcn_ssrc_register_operand (x1, DImode)))
+ return false;
+ return true;
+ }
+ else if (GET_CODE (x1) == CONST_VECTOR
+ && GET_CODE (CONST_VECTOR_ELT (x1, 0)) == CONST_INT
+ && single_cst_vector_p (x1))
+ {
+ x1 = CONST_VECTOR_ELT (x1, 0);
+ if (INTVAL (x1) >= 0 && INTVAL (x1) < (1 << 20))
+ return true;
+ }
+ }
return false;
}
@@ -1609,10 +1691,9 @@ gcn_ds_address_p (rtx x)
address space. */
bool
-gcn_global_address_p (rtx addr)
+gcn_global_address_p (rtx addr, machine_mode data_mode, bool strict)
{
- if (gcn_address_register_p (addr, DImode, false)
- || gcn_vec_address_register_p (addr, DImode, false))
+ if (gcn_auto_address_register_p (addr, data_mode, strict))
return true;
if (GET_CODE (addr) == PLUS)
@@ -1624,23 +1705,46 @@ gcn_global_address_p (rtx addr)
&& INTVAL (offset) >= -(1 << offsetbits)
&& INTVAL (offset) < (1 << offsetbits));
- if ((gcn_address_register_p (base, DImode, false)
- || gcn_vec_address_register_p (base, DImode, false))
+ if (gcn_auto_address_register_p (base, data_mode, strict)
&& immediate_p)
/* SGPR + CONST or VGPR + CONST */
return true;
- if (gcn_address_register_p (base, DImode, false)
+ if (gcn_scalar_address_register_p (base, DImode, strict)
&& gcn_vgpr_register_operand (offset, SImode))
/* SPGR + VGPR */
return true;
if (GET_CODE (base) == PLUS
- && gcn_address_register_p (XEXP (base, 0), DImode, false)
+ && gcn_scalar_address_register_p (XEXP (base, 0), DImode, strict)
&& gcn_vgpr_register_operand (XEXP (base, 1), SImode)
&& immediate_p)
/* (SGPR + VGPR) + CONST */
return true;
+
+ bool vec_immediate_p =
+ (GET_CODE (offset) == CONST_VECTOR
+ && gcn_constant_p (offset)
+ /* Signed 12/13-bit immediate. */
+ && INTVAL (CONST_VECTOR_ELT (offset, 0)) >= -(1 << offsetbits)
+ && INTVAL (CONST_VECTOR_ELT (offset, 0)) < (1 << offsetbits)
+ /* The low bits of the offset are ignored, even
+ when they're meant to realign the pointer. */
+ && !(INTVAL (CONST_VECTOR_ELT (offset, 0)) & 0x3));
+
+ if (gcn_vec_address_register_p (base, DImode, strict)
+ && vec_immediate_p)
+ /* VGPR + CONST */
+ return true;
+
+ if (GET_CODE (base) == PLUS
+ && GET_CODE (XEXP (base, 0)) == VEC_DUPLICATE
+ && gcn_scalar_address_register_p (XEXP (XEXP (base, 0), 0),
+ DImode, strict)
+ && gcn_vgpr_register_operand (XEXP (base, 1), V64SImode)
+ && vec_immediate_p)
+ /* (SGPR + VGPR) + CONST */
+ return true;
}
return false;
@@ -1660,167 +1764,27 @@ static bool
gcn_addr_space_legitimate_address_p (machine_mode mode, rtx x, bool strict,
addr_space_t as, code_helper = ERROR_MARK)
{
+ machine_mode addr_mode = GET_MODE (x);
+ if (VECTOR_MODE_P (addr_mode)
+ && (!VECTOR_MODE_P (mode)
+ || GET_MODE_NUNITS (mode) != GET_MODE_NUNITS (addr_mode)))
+ return false;
+
if (AS_SCALAR_FLAT_P (as))
{
if (mode == QImode || mode == HImode)
return 0;
-
- switch (GET_CODE (x))
- {
- case REG:
- return gcn_address_register_p (x, DImode, strict);
- /* Addresses are in the form BASE+OFFSET
- OFFSET is either 20bit unsigned immediate, SGPR or M0.
- Writes and atomics do not accept SGPR. */
- case PLUS:
- {
- rtx x0 = XEXP (x, 0);
- rtx x1 = XEXP (x, 1);
- if (!gcn_address_register_p (x0, DImode, strict))
- return false;
- /* FIXME: This is disabled because of the mode mismatch between
- SImode (for the address or m0 register) and the DImode PLUS.
- We'll need a zero_extend or similar.
-
- if (gcn_m0_register_p (x1, SImode, strict)
- || gcn_address_register_p (x1, SImode, strict))
- return true;
- else*/
- if (GET_CODE (x1) == CONST_INT)
- {
- if (INTVAL (x1) >= 0 && INTVAL (x1) < (1 << 20)
- /* The low bits of the offset are ignored, even when
- they're meant to realign the pointer. */
- && !(INTVAL (x1) & 0x3))
- return true;
- }
- return false;
- }
-
- default:
- break;
- }
+ else
+ return gcn_scalar_flat_address_p (x, strict);
}
else if (AS_SCRATCH_P (as))
- return gcn_address_register_p (x, SImode, strict);
+ return gcn_scalar_address_register_p (x, SImode, strict);
else if (AS_FLAT_P (as) || AS_FLAT_SCRATCH_P (as))
- {
- if (GET_CODE (x) == REG)
- return ((GET_MODE_CLASS (mode) == MODE_VECTOR_INT
- || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT)
- ? gcn_address_register_p (x, DImode, strict)
- : gcn_vec_address_register_p (x, DImode, strict));
- else
- {
- if (GET_CODE (x) == PLUS)
- {
- rtx x1 = XEXP (x, 1);
-
- if (VECTOR_MODE_P (mode)
- ? !gcn_address_register_p (x, DImode, strict)
- : !gcn_vec_address_register_p (x, DImode, strict))
- return false;
-
- if (GET_CODE (x1) == CONST_INT)
- {
- if (INTVAL (x1) >= 0 && INTVAL (x1) < (1 << 12)
- /* The low bits of the offset are ignored, even when
- they're meant to realign the pointer. */
- && !(INTVAL (x1) & 0x3))
- return true;
- }
- }
- return false;
- }
- }
+ return gcn_flat_address_p (x, mode, strict);
else if (AS_GLOBAL_P (as))
- {
- if (GET_CODE (x) == REG)
- return (gcn_address_register_p (x, DImode, strict)
- || (!VECTOR_MODE_P (mode)
- && gcn_vec_address_register_p (x, DImode, strict)));
- else if (GET_CODE (x) == PLUS)
- {
- rtx base = XEXP (x, 0);
- rtx offset = XEXP (x, 1);
-
- int offsetbits = (TARGET_11BIT_GLOBAL_OFFSET ? 11 : 12);
- bool immediate_p = (GET_CODE (offset) == CONST_INT
- /* Signed 12/13-bit immediate. */
- && INTVAL (offset) >= -(1 << offsetbits)
- && INTVAL (offset) < (1 << offsetbits)
- /* The low bits of the offset are ignored, even
- when they're meant to realign the pointer. */
- && !(INTVAL (offset) & 0x3));
-
- if (!VECTOR_MODE_P (mode))
- {
- if ((gcn_address_register_p (base, DImode, strict)
- || gcn_vec_address_register_p (base, DImode, strict))
- && immediate_p)
- /* SGPR + CONST or VGPR + CONST */
- return true;
-
- if (gcn_address_register_p (base, DImode, strict)
- && gcn_vgpr_register_operand (offset, SImode))
- /* SGPR + VGPR */
- return true;
-
- if (GET_CODE (base) == PLUS
- && gcn_address_register_p (XEXP (base, 0), DImode, strict)
- && gcn_vgpr_register_operand (XEXP (base, 1), SImode)
- && immediate_p)
- /* (SGPR + VGPR) + CONST */
- return true;
- }
- else
- {
- if (gcn_address_register_p (base, DImode, strict)
- && immediate_p)
- /* SGPR + CONST */
- return true;
- }
- }
- else
- return false;
- }
+ return gcn_global_address_p (x, mode, strict);
else if (AS_ANY_DS_P (as))
- switch (GET_CODE (x))
- {
- case REG:
- return (VECTOR_MODE_P (mode)
- ? gcn_address_register_p (x, SImode, strict)
- : gcn_vec_address_register_p (x, SImode, strict));
- /* Addresses are in the form BASE+OFFSET
- OFFSET is either 20bit unsigned immediate, SGPR or M0.
- Writes and atomics do not accept SGPR. */
- case PLUS:
- {
- rtx x0 = XEXP (x, 0);
- rtx x1 = XEXP (x, 1);
- if (!gcn_vec_address_register_p (x0, DImode, strict))
- return false;
- if (GET_CODE (x1) == REG)
- {
- if (GET_CODE (x1) != REG
- || (REGNO (x1) <= FIRST_PSEUDO_REGISTER
- && !gcn_ssrc_register_operand (x1, DImode)))
- return false;
- }
- else if (GET_CODE (x1) == CONST_VECTOR
- && GET_CODE (CONST_VECTOR_ELT (x1, 0)) == CONST_INT
- && single_cst_vector_p (x1))
- {
- x1 = CONST_VECTOR_ELT (x1, 0);
- if (INTVAL (x1) >= 0 && INTVAL (x1) < (1 << 20))
- return true;
- }
- return false;
- }
-
- default:
- break;
- }
+ return gcn_ds_address_p (x, strict);
else
gcc_unreachable ();
return false;
@@ -1859,6 +1823,32 @@ gcn_addr_space_address_mode (addr_space_t addrspace)
return gcn_addr_space_pointer_mode (addrspace);
}
+/* Implement TARGET_ADDR_SPACE_VALID_POINTER_MODE.
+
+ Return true if MODE is an appropriate mode for ADDRSPACE. */
+
+static bool
+gcn_addr_space_valid_pointer_mode (machine_mode mode, addr_space_t addrspace)
+{
+ if (VECTOR_MODE_P (mode))
+ mode = GET_MODE_INNER (mode);
+
+ switch (addrspace)
+ {
+ case ADDR_SPACE_SCRATCH:
+ case ADDR_SPACE_LDS:
+ case ADDR_SPACE_GDS:
+ return mode == SImode;
+ case ADDR_SPACE_DEFAULT:
+ case ADDR_SPACE_FLAT:
+ case ADDR_SPACE_FLAT_SCRATCH:
+ case ADDR_SPACE_SCALAR_FLAT:
+ return mode == DImode;
+ default:
+ gcc_unreachable ();
+ }
+}
+
/* Implement TARGET_ADDR_SPACE_SUBSET_P.
Determine if one named address space is a subset of another. */
@@ -1995,37 +1985,85 @@ gcn_regno_mode_code_ok_for_base_p (int regno,
return false;
}
-/* Implement MODE_CODE_BASE_REG_CLASS via gcn.h.
+/* Shared code for MODE_CODE_BASE_REG_CLASS and INSN_BASE_REG_CLASS. */
- Return a suitable register class for memory addressing. */
-
-reg_class
-gcn_mode_code_base_reg_class (machine_mode mode, addr_space_t as, int oc,
- int ic)
+static reg_class
+gcn_base_reg_class (machine_mode access_mode, addr_space_t as,
+ bool vector_base_p)
{
switch (as)
{
case ADDR_SPACE_DEFAULT:
- return gcn_mode_code_base_reg_class (mode, DEFAULT_ADDR_SPACE, oc, ic);
+ return gcn_base_reg_class (access_mode, DEFAULT_ADDR_SPACE, vector_base_p);
case ADDR_SPACE_SCALAR_FLAT:
case ADDR_SPACE_SCRATCH:
+ gcc_assert (!vector_base_p);
return SGPR_REGS;
break;
case ADDR_SPACE_FLAT:
case ADDR_SPACE_FLAT_SCRATCH:
case ADDR_SPACE_LDS:
case ADDR_SPACE_GDS:
- return ((GET_MODE_CLASS (mode) == MODE_VECTOR_INT
- || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT)
- ? SGPR_REGS : VGPR_REGS);
case ADDR_SPACE_GLOBAL:
- return ((GET_MODE_CLASS (mode) == MODE_VECTOR_INT
- || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT)
- ? SGPR_REGS : ALL_GPR_REGS);
+ if (vector_base_p)
+ return VGPR_REGS;
+ else if (GET_MODE_CLASS (access_mode) == MODE_VECTOR_INT
+ || GET_MODE_CLASS (access_mode) == MODE_VECTOR_FLOAT)
+ return SGPR_REGS;
+ else if (as == ADDR_SPACE_GLOBAL)
+ return ALL_GPR_REGS;
+ else
+ return VGPR_REGS;
}
gcc_unreachable ();
}
+/* Implement MODE_CODE_BASE_REG_CLASS via gcn.h.
+
+ Return a suitable register class for memory addressing of scalar
+ address modes. This is only called when INSN_BASE_REG_CLASS can't. */
+
+reg_class
+gcn_mode_code_base_reg_class (machine_mode mode, addr_space_t as, int, int)
+{
+ return gcn_base_reg_class (mode, as, false);
+}
+
+/* Implement INSN_BASE_REG_CLASS via gcn.h.
+
+ Return a suitable register class for memory addressing when the INSN is
+ known. Supports both scalar and vector addressing modes. */
+
+reg_class
+gcn_insn_base_reg_class (rtx_insn *insn)
+{
+ gcc_assert (insn);
+
+ const_rtx mem = NULL_RTX;
+ subrtx_iterator::array_type array;
+ FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
+ {
+ if (MEM_P (*iter))
+ {
+ mem = *iter;
+
+ /* There might be multiple MEM in insn, but we don't know which
+ one the caller is focussed on. It's safer to select the first
+ vector one. */
+ if (VECTOR_MODE_P (GET_MODE (XEXP (mem, 0))))
+ break;
+ }
+ }
+ if (!mem)
+ return VGPR_REGS;
+
+ machine_mode access_mode = GET_MODE (mem);
+ bool vector_base_p = VECTOR_MODE_P (GET_MODE (XEXP (mem, 0)));
+ addr_space_t as = MEM_ADDR_SPACE (mem);
+
+ return gcn_base_reg_class (access_mode, as, vector_base_p);
+}
+
/* Implement REGNO_OK_FOR_INDEX_P via gcn.h.
Return true if REGNO is OK for index of memory addressing. */
@@ -2130,12 +2168,9 @@ gcn_expand_vector_init (rtx op0, rtx vec)
if (mem_mask)
{
- emit_insn (gen_gathervNm_expr
- (op0, gen_rtx_PLUS (addrmode, addr,
- gen_rtx_VEC_DUPLICATE (addrmode,
- const0_rtx)),
- GEN_INT (DEFAULT_ADDR_SPACE), GEN_INT (0),
- NULL, get_exec (mem_mask)));
+ rtx mem = gen_rtx_MEM (mode, addr);
+ emit_insn (gen_mov_exec (mode, op0, mem, gcn_gen_undef (mode),
+ get_exec (mem_mask)));
prev = op0;
initialized_mask = mem_mask;
}
@@ -2344,10 +2379,14 @@ gcn_expand_scalar_to_vector_address (machine_mode mode, rtx exec, rtx mem,
gen_rtx_SIGN_EXTEND (pmode, tmplo));
}
- return gen_rtx_PLUS (GET_MODE (new_base), new_base,
- gen_rtx_VEC_DUPLICATE (GET_MODE (new_base),
- (mem_index ? mem_index
- : const0_rtx)));
+ rtx addr = gen_rtx_PLUS (GET_MODE (new_base), new_base,
+ gen_rtx_VEC_DUPLICATE (GET_MODE (new_base),
+ (mem_index ? mem_index
+ : const0_rtx)));
+ rtx newmem = gen_rtx_MEM (mode, addr);
+ set_mem_addr_space (newmem, MEM_ADDR_SPACE (mem));
+ MEM_VOLATILE_P (newmem) = MEM_VOLATILE_P (mem);
+ return newmem;
}
/* Convert a BASE address, a vector of OFFSETS, and a SCALE, to addresses
@@ -2403,6 +2442,21 @@ gcn_expand_scaled_offsets (addr_space_t as, rtx base, rtx offsets, rtx scale,
gcc_unreachable ();
}
+/* Convert gather/scatter parameters to a vector MEM. */
+
+rtx
+gcn_gen_vector_mem (machine_mode mode, addr_space_t as, rtx scalarbase,
+ rtx vectoroffsets, rtx scale, bool unsigned_p,
+ bool volatile_p, rtx exec)
+{
+ rtx addr = gcn_expand_scaled_offsets (as, scalarbase, vectoroffsets, scale,
+ unsigned_p, exec);
+ rtx mem = gen_rtx_MEM (mode, addr);
+ set_mem_addr_space (mem, as);
+ MEM_VOLATILE_P (mem) = volatile_p;
+ return mem;
+}
+
/* Return true if move from OP0 to OP1 is known to be executed in vector
unit. */
@@ -2494,8 +2548,9 @@ gcn_secondary_reload (bool in_p, rtx x, reg_class_t rclass,
case ADDR_SPACE_FLAT:
case ADDR_SPACE_FLAT_SCRATCH:
case ADDR_SPACE_GLOBAL:
- if (GET_MODE_CLASS (reload_mode) == MODE_VECTOR_INT
- || GET_MODE_CLASS (reload_mode) == MODE_VECTOR_FLOAT)
+ if ((GET_MODE_CLASS (reload_mode) == MODE_VECTOR_INT
+ || GET_MODE_CLASS (reload_mode) == MODE_VECTOR_FLOAT)
+ && !(MEM_P (x) && VECTOR_MODE_P (GET_MODE (XEXP (x, 0)))))
{
sri->icode = code_for_mov_sgprbase (reload_mode);
break;
@@ -2659,14 +2714,14 @@ gcn_valid_move_p (machine_mode mode, rtx dest, rtx src)
if (MEM_P (dest)
&& AS_GLOBAL_P (MEM_ADDR_SPACE (dest))
- && (gcn_global_address_p (XEXP (dest, 0))
+ && (gcn_global_address_p (XEXP (dest, 0), GET_MODE (dest))
|| GET_CODE (XEXP (dest, 0)) == SYMBOL_REF
|| GET_CODE (XEXP (dest, 0)) == LABEL_REF)
&& gcn_vgpr_equivalent_register_operand (src, mode))
return true;
else if (MEM_P (src)
&& AS_GLOBAL_P (MEM_ADDR_SPACE (src))
- && (gcn_global_address_p (XEXP (src, 0))
+ && (gcn_global_address_p (XEXP (src, 0), GET_MODE (src))
|| GET_CODE (XEXP (src, 0)) == SYMBOL_REF
|| GET_CODE (XEXP (src, 0)) == LABEL_REF)
&& gcn_vgpr_equivalent_register_operand (dest, mode))
@@ -3154,7 +3209,6 @@ move_callee_saved_registers (rtx sp, machine_function *offsets,
rtx exec = gen_rtx_REG (DImode, EXEC_REG);
rtx vcc = gen_rtx_REG (DImode, VCC_LO_REG);
rtx offreg = gen_rtx_REG (SImode, SGPR_REGNO (22));
- rtx as = gen_rtx_CONST_INT (VOIDmode, STACK_ADDR_SPACE);
HOST_WIDE_INT exec_set = 0;
int offreg_set = 0;
auto_vec<int> saved_sgprs;
@@ -3207,6 +3261,8 @@ move_callee_saved_registers (rtx sp, machine_function *offsets,
gcn_operand_part (V64SImode, vsp, 1),
const0_rtx, vcc, vcc,
gcn_gen_undef (V64SImode), exec));
+ rtx vspmem = gen_rtx_MEM (V64SImode, vsp);
+ set_mem_addr_space (vspmem, STACK_ADDR_SPACE);
/* Move vectors. */
for (regno = FIRST_VGPR_REG, offset = 0;
@@ -3231,9 +3287,9 @@ move_callee_saved_registers (rtx sp, machine_function *offsets,
if (prologue)
{
- rtx insn = emit_insn (gen_scatterv64si_insn_1offset_exec
- (vsp, const0_rtx, reg, as, const0_rtx,
- exec));
+ rtx insn = emit_insn (gen_movv64si_exec (vspmem, reg,
+ gcn_gen_undef (V64SImode),
+ exec));
/* Add CFI metadata. */
rtx note;
@@ -3292,9 +3348,8 @@ move_callee_saved_registers (rtx sp, machine_function *offsets,
add_reg_note (insn, REG_FRAME_RELATED_EXPR, note);
}
else
- emit_insn (gen_gatherv64si_insn_1offset_exec
- (reg, vsp, const0_rtx, as, const0_rtx,
- gcn_gen_undef (V64SImode), exec));
+ emit_insn (gen_movv64si_exec (reg, vspmem, gcn_gen_undef (V64SImode),
+ exec));
/* Move our VSP to the next stack entry. */
if (offreg_set != size)
@@ -7230,6 +7285,7 @@ print_operand_address (FILE *file, rtx mem)
rtx offset;
addr_space_t as = MEM_ADDR_SPACE (mem);
rtx addr = XEXP (mem, 0);
+
gcc_assert (REG_P (addr) || GET_CODE (addr) == PLUS);
if (AS_SCRATCH_P (as))
@@ -7271,9 +7327,13 @@ print_operand_address (FILE *file, rtx mem)
if (GET_CODE (base) == PLUS)
{
- /* (SGPR + VGPR) + CONST */
+ /* (SGPR + VGPR) + CONST
+ Note: the offset is printed by %O. */
vgpr_offset = XEXP (base, 1);
base = XEXP (base, 0);
+
+ if (GET_CODE (base) == VEC_DUPLICATE)
+ base = XEXP (base, 0);
}
else
{
@@ -7309,8 +7369,6 @@ print_operand_address (FILE *file, rtx mem)
output_operand_lossage ("bad ADDR_SPACE_GLOBAL address");
}
}
- else
- output_operand_lossage ("bad ADDR_SPACE_GLOBAL address");
}
else if (AS_ANY_DS_P (as))
switch (GET_CODE (addr))
@@ -7640,12 +7698,19 @@ print_operand (FILE *file, rtx x, int code)
base = XEXP (x0, 0);
if (GET_CODE (base) == PLUS)
- /* (SGPR + VGPR) + CONST */
- /* Ignore the VGPR offset for this operand. */
- base = XEXP (base, 0);
+ {
+ /* (SGPR + VGPR) + CONST */
+ /* Ignore the VGPR offset for this operand. */
+ base = XEXP (base, 0);
+
+ if (GET_CODE (base) == VEC_DUPLICATE)
+ base = XEXP (base, 0);
+ }
+ if (CONST_VECTOR_P (offset))
+ offset = CONST_VECTOR_ELT (offset, 0);
if (CONST_INT_P (offset))
- const_offset = XEXP (x0, 1);
+ const_offset = offset;
else if (REG_P (offset))
/* SGPR + VGPR */
/* Ignore the VGPR offset for this operand. */
@@ -7684,6 +7749,8 @@ print_operand (FILE *file, rtx x, int code)
rtx val = XEXP (x0, 1);
if (GET_CODE (val) == CONST_VECTOR)
val = CONST_VECTOR_ELT (val, 0);
+ if (GET_CODE (val) == VEC_DUPLICATE)
+ val = XEXP (val, 0);
if (GET_CODE (val) != CONST_INT)
{
output_operand_lossage ("invalid %%xn code");
@@ -8097,6 +8164,8 @@ gcn_dwarf_register_span (rtx rtl)
#define TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID gcn_addr_space_zero_address_valid
#undef TARGET_ADDR_SPACE_CONVERT
#define TARGET_ADDR_SPACE_CONVERT gcn_addr_space_convert
+#undef TARGET_ADDR_SPACE_VALID_POINTER_MODE
+#define TARGET_ADDR_SPACE_VALID_POINTER_MODE gcn_addr_space_valid_pointer_mode
#undef TARGET_ARG_PARTIAL_BYTES
#define TARGET_ARG_PARTIAL_BYTES gcn_arg_partial_bytes
#undef TARGET_ASM_ALIGNED_DI_OP
@@ -602,6 +602,7 @@ enum reg_class
#define REGNO_REG_CLASS(REGNO) gcn_regno_reg_class (REGNO)
#define MODE_CODE_BASE_REG_CLASS(MODE, AS, OUTER, INDEX) \
gcn_mode_code_base_reg_class (MODE, AS, OUTER, INDEX)
+#define INSN_BASE_REG_CLASS(INSN) gcn_insn_base_reg_class (INSN)
#define REGNO_MODE_CODE_OK_FOR_BASE_P(NUM, MODE, AS, OUTER, INDEX) \
gcn_regno_mode_code_ok_for_base_p (NUM, MODE, AS, OUTER, INDEX)
#define INDEX_REG_CLASS VGPR_REGS
@@ -67,6 +67,7 @@ (define_c_enum "unspec" [
UNSPEC_VECTOR
UNSPEC_BPERMUTE
UNSPEC_SGPRBASE
+ UNSPEC_MASKLOAD
UNSPEC_MEMORY_BARRIER
UNSPEC_SMIN_DPP_SHR UNSPEC_SMAX_DPP_SHR
UNSPEC_UMIN_DPP_SHR UNSPEC_UMAX_DPP_SHR
@@ -81,8 +82,6 @@ (define_c_enum "unspec" [
UNSPEC_CMUL_ADD UNSPEC_CMUL_SUB
UNSPEC_CADD90
UNSPEC_CADD270
- UNSPEC_GATHER
- UNSPEC_SCATTER
UNSPEC_RCP
UNSPEC_FLBIT_INT
UNSPEC_FLOOR UNSPEC_CEIL UNSPEC_SIN UNSPEC_COS UNSPEC_EXP2 UNSPEC_LOG2
@@ -1976,7 +1975,7 @@ (define_insn "atomic_fetch_<bare_mnemonic><mode>"
"0 /* Disabled. */"
"@
s_atomic_<bare_mnemonic><X>\t%0, %1, %2 glc\;s_waitcnt\tlgkmcnt(0)
- flat_atomic_<bare_mnemonic><X>\t%0, %1, %2 %G2\;s_waitcnt\t0
+ flat_atomic_<bare_mnemonic><X>\t%0, %1, %2%O1 %G2\;s_waitcnt\t0
global_atomic_<bare_mnemonic><X>\t%0, %A1, %2%O1 %G2\;s_waitcnt\tvmcnt(0)"
[(set_attr "type" "smem,flat,flat")
(set_attr "flatmemaccess" "*,atomicwait,atomicwait")
@@ -1998,7 +1997,7 @@ (define_insn "atomic_<bare_mnemonic><mode>"
"0 /* Disabled. */"
"@
s_atomic_<bare_mnemonic><X>\t%0, %1\;s_waitcnt\tlgkmcnt(0)
- flat_atomic_<bare_mnemonic><X>\t%0, %1\;s_waitcnt\t0
+ flat_atomic_<bare_mnemonic><X>\t%0, %1%O0\;s_waitcnt\t0
global_atomic_<bare_mnemonic><X>\t%A0, %1%O0\;s_waitcnt\tvmcnt(0)"
[(set_attr "type" "smem,flat,flat")
(set_attr "flatmemaccess" "*,atomicwait,atomicwait")
@@ -2045,7 +2044,7 @@ (define_insn "sync_compare_and_swap<mode>_insn"
""
"@
s_atomic_cmpswap<X>\t%0, %1, %2 glc\;s_waitcnt\tlgkmcnt(0)
- flat_atomic_cmpswap<X>\t%0, %1, %2 %G2\;s_waitcnt\t0
+ flat_atomic_cmpswap<X>\t%0, %1, %2%O1 %G2\;s_waitcnt\t0
global_atomic_cmpswap<X>\t%0, %A1, %2%O1 %G2\;s_waitcnt\tvmcnt(0)"
[(set_attr "type" "smem,flat,flat")
(set_attr "length" "12")