@@ -275,18 +275,20 @@ parse_signature (const function_instance &instance, const char *format,
}
/* Add one function instance for GROUP, using mode suffix MODE_SUFFIX_ID,
- the type suffixes at index TI and the predication suffix at index PI.
- The other arguments are as for build_all. */
+ the type suffixes at index TI, the group suffixes at index GI, and the
+ predication suffix at index PI. The other arguments are as for
+ build_all. */
static void
build_one (function_builder &b, const char *signature,
const function_group_info &group, mode_suffix_index mode_suffix_id,
- unsigned int ti, unsigned int pi, bool force_direct_overloads)
+ unsigned int ti, unsigned int gi, unsigned int pi,
+ bool force_direct_overloads)
{
/* Byte forms of svdupq take 16 arguments. */
auto_vec<tree, 16> argument_types;
function_instance instance (group.base_name, *group.base, *group.shape,
mode_suffix_id, group.types[ti],
- group.preds[pi]);
+ group.groups[gi], group.preds[pi]);
tree return_type = parse_signature (instance, signature, argument_types);
apply_predication (instance, return_type, argument_types);
b.add_unique_function (instance, return_type, argument_types,
@@ -312,24 +314,26 @@ build_32_64 (function_builder &b, const char *signature,
mode_suffix_index mode64, bool force_direct_overloads = false)
{
for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
- if (group.types[0][0] == NUM_TYPE_SUFFIXES)
- {
- gcc_assert (mode32 != MODE_none && mode64 != MODE_none);
- build_one (b, signature, group, mode32, 0, pi,
- force_direct_overloads);
- build_one (b, signature, group, mode64, 0, pi,
- force_direct_overloads);
- }
- else
- for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES; ++ti)
+ for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
+ if (group.types[0][0] == NUM_TYPE_SUFFIXES)
{
- unsigned int bits = type_suffixes[group.types[ti][0]].element_bits;
- gcc_assert (bits == 32 || bits == 64);
- mode_suffix_index mode = bits == 32 ? mode32 : mode64;
- if (mode != MODE_none)
- build_one (b, signature, group, mode, ti, pi,
- force_direct_overloads);
+ gcc_assert (mode32 != MODE_none && mode64 != MODE_none);
+ build_one (b, signature, group, mode32, 0, gi, pi,
+ force_direct_overloads);
+ build_one (b, signature, group, mode64, 0, gi, pi,
+ force_direct_overloads);
}
+ else
+ for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES;
+ ++ti)
+ {
+ unsigned int bits = type_suffixes[group.types[ti][0]].element_bits;
+ gcc_assert (bits == 32 || bits == 64);
+ mode_suffix_index mode = bits == 32 ? mode32 : mode64;
+ if (mode != MODE_none)
+ build_one (b, signature, group, mode, ti, gi, pi,
+ force_direct_overloads);
+ }
}
/* For every type and predicate combination in GROUP, add one function
@@ -423,10 +427,11 @@ build_all (function_builder &b, const char *signature,
bool force_direct_overloads = false)
{
for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
- for (unsigned int ti = 0;
- ti == 0 || group.types[ti][0] != NUM_TYPE_SUFFIXES; ++ti)
- build_one (b, signature, group, mode_suffix_id, ti, pi,
- force_direct_overloads);
+ for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
+ for (unsigned int ti = 0;
+ ti == 0 || group.types[ti][0] != NUM_TYPE_SUFFIXES; ++ti)
+ build_one (b, signature, group, mode_suffix_id, ti, gi, pi,
+ force_direct_overloads);
}
/* TYPE is the largest type suffix associated with the arguments of R,
@@ -247,7 +247,7 @@ public:
that we can use for sensible shift amounts. */
function_instance instance ("svqshl", functions::svqshl,
shapes::binary_int_opt_n, MODE_n,
- f.type_suffix_ids, f.pred);
+ f.type_suffix_ids, GROUP_none, f.pred);
return f.redirect_call (instance);
}
else
@@ -256,7 +256,7 @@ public:
that we can use for sensible shift amounts. */
function_instance instance ("svrshl", functions::svrshl,
shapes::binary_int_opt_n, MODE_n,
- f.type_suffix_ids, f.pred);
+ f.type_suffix_ids, GROUP_none, f.pred);
return f.redirect_call (instance);
}
}
@@ -285,7 +285,7 @@ public:
-wi::to_wide (amount));
function_instance instance ("svasr", functions::svasr,
shapes::binary_uint_opt_n, MODE_n,
- f.type_suffix_ids, f.pred);
+ f.type_suffix_ids, GROUP_none, f.pred);
if (f.type_suffix (0).unsigned_p)
{
instance.base_name = "svlsr";
@@ -317,7 +317,7 @@ public:
that we can use for sensible shift amounts. */
function_instance instance ("svlsl", functions::svlsl,
shapes::binary_uint_opt_n, MODE_n,
- f.type_suffix_ids, f.pred);
+ f.type_suffix_ids, GROUP_none, f.pred);
gcall *call = as_a <gcall *> (f.redirect_call (instance));
gimple_call_set_arg (call, 2, amount);
return call;
@@ -330,7 +330,7 @@ public:
-wi::to_wide (amount));
function_instance instance ("svrshr", functions::svrshr,
shapes::shift_right_imm, MODE_n,
- f.type_suffix_ids, f.pred);
+ f.type_suffix_ids, GROUP_none, f.pred);
gcall *call = as_a <gcall *> (f.redirect_call (instance));
gimple_call_set_arg (call, 2, amount);
return call;
@@ -144,6 +144,13 @@ CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = {
0, VOIDmode }
};
+CONSTEXPR const group_suffix_info group_suffixes[] = {
+#define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) \
+ { "_" #NAME, VG, VECTORS_PER_TUPLE },
+#include "aarch64-sve-builtins.def"
+ { "", 0, 1 }
+};
+
/* Define a TYPES_<combination> macro for each combination of type
suffixes that an ACLE function can have, where <combination> is the
name used in DEF_SVE_FUNCTION entries.
@@ -483,6 +490,10 @@ DEF_SVE_TYPES_ARRAY (inc_dec_n);
DEF_SVE_TYPES_ARRAY (reinterpret);
DEF_SVE_TYPES_ARRAY (while);
+static const group_suffix_index groups_none[] = {
+ GROUP_none, NUM_GROUP_SUFFIXES
+};
+
/* Used by functions that have no governing predicate. */
static const predication_index preds_none[] = { PRED_none, NUM_PREDS };
@@ -524,8 +535,8 @@ static const predication_index preds_z[] = { PRED_z, NUM_PREDS };
/* A list of all SVE ACLE functions. */
static CONSTEXPR const function_group_info function_groups[] = {
#define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \
- { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, preds_##PREDS, \
- REQUIRED_EXTENSIONS },
+ { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, groups_none, \
+ preds_##PREDS, REQUIRED_EXTENSIONS },
#include "aarch64-sve-builtins.def"
};
@@ -788,6 +799,7 @@ function_instance::hash () const
h.add_int (mode_suffix_id);
h.add_int (type_suffix_ids[0]);
h.add_int (type_suffix_ids[1]);
+ h.add_int (group_suffix_id);
h.add_int (pred);
return h.end ();
}
@@ -957,6 +969,8 @@ function_builder::get_name (const function_instance &instance,
for (unsigned int i = 0; i < 2; ++i)
if (!overloaded_p || instance.shape->explicit_type_suffix_p (i))
append_name (instance.type_suffix (i).string);
+ if (!overloaded_p || instance.shape->explicit_group_suffix_p ())
+ append_name (instance.group_suffix ().string);
append_name (pred_suffixes[instance.pred]);
return finish_name ();
}
@@ -1113,19 +1127,26 @@ void
function_builder::add_overloaded_functions (const function_group_info &group,
mode_suffix_index mode)
{
- unsigned int explicit_type0 = (*group.shape)->explicit_type_suffix_p (0);
- unsigned int explicit_type1 = (*group.shape)->explicit_type_suffix_p (1);
- for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
+ bool explicit_type0 = (*group.shape)->explicit_type_suffix_p (0);
+ bool explicit_type1 = (*group.shape)->explicit_type_suffix_p (1);
+ bool explicit_group = (*group.shape)->explicit_group_suffix_p ();
+ auto add_function = [&](const type_suffix_pair &types,
+ group_suffix_index group_suffix_id,
+ unsigned int pi)
+ {
+ function_instance instance (group.base_name, *group.base,
+ *group.shape, mode, types,
+ group_suffix_id, group.preds[pi]);
+ add_overloaded_function (instance, group.required_extensions);
+ };
+
+ auto add_group_suffix = [&](group_suffix_index group_suffix_id,
+ unsigned int pi)
{
if (!explicit_type0 && !explicit_type1)
- {
- /* Deal with the common case in which there is one overloaded
- function for all type combinations. */
- function_instance instance (group.base_name, *group.base,
- *group.shape, mode, types_none[0],
- group.preds[pi]);
- add_overloaded_function (instance, group.required_extensions);
- }
+ /* Deal with the common case in which there is one overloaded
+ function for all type combinations. */
+ add_function (types_none[0], group_suffix_id, pi);
else
for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES;
++ti)
@@ -1136,12 +1157,16 @@ function_builder::add_overloaded_functions (const function_group_info &group,
explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES,
explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES
};
- function_instance instance (group.base_name, *group.base,
- *group.shape, mode, types,
- group.preds[pi]);
- add_overloaded_function (instance, group.required_extensions);
+ add_function (types, group_suffix_id, pi);
}
- }
+ };
+
+ for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
+ if (explicit_group)
+ for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
+ add_group_suffix (group.groups[gi], pi);
+ else
+ add_group_suffix (GROUP_none, pi);
}
/* Register all the functions in GROUP. */
@@ -1213,29 +1238,34 @@ function_resolver::report_no_such_form (type_suffix_index type)
}
/* Silently check whether there is an instance of the function with the
- mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1.
- Return its function decl if so, otherwise return null. */
+ mode suffix given by MODE, the type suffixes given by TYPE0 and TYPE1,
+ and the group suffix given by GROUP. Return its function decl if so,
+ otherwise return null. */
tree
function_resolver::lookup_form (mode_suffix_index mode,
type_suffix_index type0,
- type_suffix_index type1)
+ type_suffix_index type1,
+ group_suffix_index group)
{
type_suffix_pair types = { type0, type1 };
- function_instance instance (base_name, base, shape, mode, types, pred);
+ function_instance instance (base_name, base, shape, mode, types,
+ group, pred);
registered_function *rfn
= function_table->find_with_hash (instance, instance.hash ());
return rfn ? rfn->decl : NULL_TREE;
}
-/* Resolve the function to one with the mode suffix given by MODE and the
- type suffixes given by TYPE0 and TYPE1. Return its function decl on
- success, otherwise report an error and return error_mark_node. */
+/* Resolve the function to one with the mode suffix given by MODE, the
+ type suffixes given by TYPE0 and TYPE1, and group suffix given by
+ GROUP. Return its function decl on success, otherwise report an
+ error and return error_mark_node. */
tree
function_resolver::resolve_to (mode_suffix_index mode,
type_suffix_index type0,
- type_suffix_index type1)
+ type_suffix_index type1,
+ group_suffix_index group)
{
- tree res = lookup_form (mode, type0, type1);
+ tree res = lookup_form (mode, type0, type1, group);
if (!res)
{
if (type1 == NUM_TYPE_SUFFIXES)
@@ -29,6 +29,10 @@
#define DEF_SVE_TYPE_SUFFIX(A, B, C, D, E)
#endif
+#ifndef DEF_SVE_GROUP_SUFFIX
+#define DEF_SVE_GROUP_SUFFIX(A, B, C)
+#endif
+
#ifndef DEF_SVE_FUNCTION
#define DEF_SVE_FUNCTION(A, B, C, D)
#endif
@@ -95,10 +99,15 @@ DEF_SVE_TYPE_SUFFIX (u16, svuint16_t, unsigned, 16, VNx8HImode)
DEF_SVE_TYPE_SUFFIX (u32, svuint32_t, unsigned, 32, VNx4SImode)
DEF_SVE_TYPE_SUFFIX (u64, svuint64_t, unsigned, 64, VNx2DImode)
+DEF_SVE_GROUP_SUFFIX (x2, 0, 2)
+DEF_SVE_GROUP_SUFFIX (x3, 0, 3)
+DEF_SVE_GROUP_SUFFIX (x4, 0, 4)
+
#include "aarch64-sve-builtins-base.def"
#include "aarch64-sve-builtins-sve2.def"
#undef DEF_SVE_FUNCTION
+#undef DEF_SVE_GROUP_SUFFIX
#undef DEF_SVE_TYPE_SUFFIX
#undef DEF_SVE_TYPE
#undef DEF_SVE_MODE
@@ -180,6 +180,17 @@ enum type_suffix_index
NUM_TYPE_SUFFIXES
};
+/* Enumerates the possible group suffixes. Each suffix combines two
+ optional pieces of information: the vector group size in a ZA index,
+ and the number of vectors in the largest tuple argument. */
+enum group_suffix_index
+{
+#define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) GROUP_##NAME,
+#include "aarch64-sve-builtins.def"
+ GROUP_none,
+ NUM_GROUP_SUFFIXES
+};
+
/* Combines two type suffixes. */
typedef enum type_suffix_index type_suffix_pair[2];
@@ -237,6 +248,21 @@ struct type_suffix_info
machine_mode vector_mode : 16;
};
+/* Static information about a group suffix. */
+struct group_suffix_info
+{
+ /* The suffix string itself. */
+ const char *string;
+
+ /* If the suffix describes a vector group in a ZA index, this is the
+ size of that group, otherwise it is zero. */
+ unsigned int vg;
+
+ /* The number of vectors in the largest (or only) tuple argument,
+ or 1 if the suffix does not convey this information. */
+ unsigned int vectors_per_tuple;
+};
+
/* Static information about a set of functions. */
struct function_group_info
{
@@ -251,14 +277,16 @@ struct function_group_info
shapes. */
const function_shape *const *shape;
- /* A list of the available type suffixes, and of the available predication
- types. The function supports every combination of the two.
+ /* A list of the available type suffixes, group suffixes, and predication
+ types. The function supports every combination of the three.
+
+ The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES.
+ It is lexicographically ordered based on the index value.
- The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES
- while the list of predication types is terminated by NUM_PREDS.
- The list of type suffixes is lexicographically ordered based
- on the index value. */
+ The list of group suffixes is terminated by NUM_GROUP_SUFFIXES
+ and the list of predication types is terminated by NUM_PREDS. */
const type_suffix_pair *types;
+ const group_suffix_index *groups;
const predication_index *preds;
/* The architecture extensions that the functions require, as a set of
@@ -273,7 +301,8 @@ class GTY((user)) function_instance
public:
function_instance (const char *, const function_base *,
const function_shape *, mode_suffix_index,
- const type_suffix_pair &, predication_index);
+ const type_suffix_pair &, group_suffix_index,
+ predication_index);
bool operator== (const function_instance &) const;
bool operator!= (const function_instance &) const;
@@ -294,6 +323,8 @@ public:
units_index displacement_units () const;
const type_suffix_info &type_suffix (unsigned int) const;
+ const group_suffix_info &group_suffix () const;
+
tree scalar_type (unsigned int) const;
tree vector_type (unsigned int) const;
tree tuple_type (unsigned int) const;
@@ -301,14 +332,14 @@ public:
machine_mode vector_mode (unsigned int) const;
machine_mode gp_mode (unsigned int) const;
- /* The properties of the function. (The explicit "enum"s are required
- for gengtype.) */
+ /* The properties of the function. */
const char *base_name;
const function_base *base;
const function_shape *shape;
- enum mode_suffix_index mode_suffix_id;
+ mode_suffix_index mode_suffix_id;
type_suffix_pair type_suffix_ids;
- enum predication_index pred;
+ group_suffix_index group_suffix_id;
+ predication_index pred;
};
class registered_function;
@@ -390,10 +421,12 @@ public:
tree report_no_such_form (type_suffix_index);
tree lookup_form (mode_suffix_index,
type_suffix_index = NUM_TYPE_SUFFIXES,
- type_suffix_index = NUM_TYPE_SUFFIXES);
+ type_suffix_index = NUM_TYPE_SUFFIXES,
+ group_suffix_index = GROUP_none);
tree resolve_to (mode_suffix_index,
type_suffix_index = NUM_TYPE_SUFFIXES,
- type_suffix_index = NUM_TYPE_SUFFIXES);
+ type_suffix_index = NUM_TYPE_SUFFIXES,
+ group_suffix_index = GROUP_none);
type_suffix_index infer_integer_scalar_type (unsigned int);
type_suffix_index infer_pointer_type (unsigned int, bool = false);
@@ -641,6 +674,11 @@ class function_shape
public:
virtual bool explicit_type_suffix_p (unsigned int) const = 0;
+ /* True if the group suffix is present in overloaded names.
+ This isn't meaningful for pre-SME intrinsics, and true is
+ more common than false, so provide a default definition. */
+ virtual bool explicit_group_suffix_p () const { return true; }
+
/* Define all functions associated with the given group. */
virtual void build (function_builder &,
const function_group_info &) const = 0;
@@ -669,6 +707,7 @@ private:
extern const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1];
extern const mode_suffix_info mode_suffixes[MODE_none + 1];
+extern const group_suffix_info group_suffixes[NUM_GROUP_SUFFIXES];
extern tree scalar_types[NUM_VECTOR_TYPES];
extern tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1];
@@ -728,9 +767,11 @@ function_instance (const char *base_name_in,
const function_shape *shape_in,
mode_suffix_index mode_suffix_id_in,
const type_suffix_pair &type_suffix_ids_in,
+ group_suffix_index group_suffix_id_in,
predication_index pred_in)
: base_name (base_name_in), base (base_in), shape (shape_in),
- mode_suffix_id (mode_suffix_id_in), pred (pred_in)
+ mode_suffix_id (mode_suffix_id_in), group_suffix_id (group_suffix_id_in),
+ pred (pred_in)
{
memcpy (type_suffix_ids, type_suffix_ids_in, sizeof (type_suffix_ids));
}
@@ -741,9 +782,10 @@ function_instance::operator== (const function_instance &other) const
return (base == other.base
&& shape == other.shape
&& mode_suffix_id == other.mode_suffix_id
- && pred == other.pred
&& type_suffix_ids[0] == other.type_suffix_ids[0]
- && type_suffix_ids[1] == other.type_suffix_ids[1]);
+ && type_suffix_ids[1] == other.type_suffix_ids[1]
+ && group_suffix_id == other.group_suffix_id
+ && pred == other.pred);
}
inline bool
@@ -815,6 +857,13 @@ function_instance::type_suffix (unsigned int i) const
return type_suffixes[type_suffix_ids[i]];
}
+/* Return information about the function's group suffix. */
+inline const group_suffix_info &
+function_instance::group_suffix () const
+{
+ return group_suffixes[group_suffix_id];
+}
+
/* Return the scalar type associated with type suffix I. */
inline tree
function_instance::scalar_type (unsigned int i) const