@@ -900,6 +900,7 @@ struct GTY((tag ("SYMTAB_FUNCTION"))) cgraph_node : public symtab_node
ipcp_clone (false), declare_variant_alt (false),
calls_declare_variant_alt (false), gc_candidate (false),
called_by_ifunc_resolver (false),
+ has_metadirectives (false),
m_uid (uid), m_summary_id (-1)
{}
@@ -1501,6 +1502,8 @@ struct GTY((tag ("SYMTAB_FUNCTION"))) cgraph_node : public symtab_node
unsigned gc_candidate : 1;
/* Set if the function is called by an IFUNC resolver. */
unsigned called_by_ifunc_resolver : 1;
+ /* True if the function contains unresolved metadirectives. */
+ unsigned has_metadirectives : 1;
private:
/* Unique id of the node. */
@@ -389,6 +389,7 @@ cgraph_node::create_clone (tree new_decl, profile_count prof_count,
prof_count = count.combine_with_ipa_count (prof_count);
new_node->count = prof_count;
new_node->calls_declare_variant_alt = this->calls_declare_variant_alt;
+ new_node->has_metadirectives = this->has_metadirectives;
/* Update IPA profile. Local profiles need no updating in original. */
if (update_original)
@@ -310,6 +310,9 @@ kinds, along with their relationships to @code{GSS_} values (layouts) and
+ gimple_statement_with_ops_base
| | (no GSS layout)
| |
+ | + gomp_metadirective
+ | | code: GIMPLE_OMP_METADIRECTIVE
+ | |
| + gimple_statement_with_ops
| | | layout: GSS_WITH_OPS
| | |
@@ -358,6 +361,9 @@ kinds, along with their relationships to @code{GSS_} values (layouts) and
| + gomp_for
| | layout: GSS_OMP_FOR, code: GIMPLE_OMP_FOR
| |
+ | + gomp_variant
+ | | code: GIMPLE_OMP_METADIRECTIVE_VARIANT
+ | |
| + gomp_parallel_layout
| | | layout: GSS_OMP_PARALLEL_LAYOUT
| | |
@@ -229,6 +229,36 @@ lower_sequence (gimple_seq *seq, struct lower_data *data)
lower_stmt (&gsi, data);
}
+/* Lower the OpenMP metadirective statement pointed by GSI. */
+
+static void
+lower_omp_metadirective (gimple_stmt_iterator *gsi, struct lower_data *data)
+{
+ gimple *stmt = gsi_stmt (*gsi);
+ gimple_seq variant_seq = gimple_omp_variants (stmt);
+ gimple_stmt_iterator variant_gsi = gsi_start (variant_seq);
+ unsigned i;
+
+ /* The variants are not used after lowering. */
+ gimple_omp_metadirective_set_variants (stmt, NULL);
+
+ for (i = 0; i < gimple_num_ops (stmt); i++)
+ {
+ gimple *variant = gsi_stmt (variant_gsi);
+ tree label = create_artificial_label (UNKNOWN_LOCATION);
+ gimple_omp_metadirective_set_label (stmt, i, label);
+ gsi_insert_after (gsi, gimple_build_label (label), GSI_CONTINUE_LINKING);
+
+ gimple_seq *directive_ptr = gimple_omp_body_ptr (variant);
+ lower_sequence (directive_ptr, data);
+ gsi_insert_seq_after (gsi, *directive_ptr, GSI_CONTINUE_LINKING);
+
+ gsi_next (&variant_gsi);
+ }
+
+ gsi_next (gsi);
+}
+
/* Lower the OpenMP directive statement pointed by GSI. DATA is
passed through the recursion. */
@@ -843,6 +873,12 @@ lower_stmt (gimple_stmt_iterator *gsi, struct lower_data *data)
lower_assumption (gsi, data);
return;
+ case GIMPLE_OMP_METADIRECTIVE:
+ data->cannot_fallthru = false;
+ lower_omp_metadirective (gsi, data);
+ data->cannot_fallthru = false;
+ return;
+
case GIMPLE_TRANSACTION:
lower_sequence (gimple_transaction_body_ptr (
as_a <gtransaction *> (stmt)),
@@ -2075,6 +2075,64 @@ dump_gimple_assume (pretty_printer *buffer, const gimple *gs,
}
}
+/* Dump a GIMPLE_OMP_METADIRECTIVE tuple on the pretty_printer BUFFER. */
+
+static void
+dump_gimple_omp_metadirective (pretty_printer *buffer, const gimple *gs,
+ int spc, dump_flags_t flags)
+{
+ if (flags & TDF_RAW)
+ dump_gimple_fmt (buffer, spc, flags, "%G <%+BODY <%S> >", gs,
+ gimple_omp_body (gs));
+ else
+ {
+ pp_string (buffer, "#pragma omp metadirective");
+ newline_and_indent (buffer, spc + 2);
+
+ gimple_seq variant_seq = gimple_omp_variants (gs);
+ gimple_stmt_iterator gsi = gsi_start (variant_seq);
+
+ for (unsigned i = 0; i < gimple_num_ops (gs); i++)
+ {
+ tree selector = gimple_op (gs, i);
+
+ if (selector == NULL_TREE)
+ pp_string (buffer, "default:");
+ else
+ {
+ pp_string (buffer, "when (");
+ dump_omp_context_selector (buffer, selector, spc, flags);
+ pp_string (buffer, "):");
+ }
+
+ gimple *variant = gsi_stmt (gsi);
+
+ if (variant != NULL)
+ {
+ newline_and_indent (buffer, spc + 4);
+ pp_left_brace (buffer);
+ pp_newline (buffer);
+ dump_gimple_seq (buffer, gimple_omp_body (variant), spc + 6,
+ flags);
+ newline_and_indent (buffer, spc + 4);
+ pp_right_brace (buffer);
+
+ gsi_next (&gsi);
+ }
+ else
+ {
+ tree label = gimple_omp_metadirective_label (gs, i);
+
+ pp_string (buffer, " ");
+ dump_generic_node (buffer, label, spc, flags, false);
+ }
+
+ if (i != gimple_num_ops (gs) - 1)
+ newline_and_indent (buffer, spc + 2);
+ }
+ }
+}
+
/* Dump a GIMPLE_TRANSACTION tuple on the pretty_printer BUFFER. */
static void
@@ -2823,6 +2881,12 @@ pp_gimple_stmt_1 (pretty_printer *buffer, const gimple *gs, int spc,
flags);
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ dump_gimple_omp_metadirective (buffer,
+ as_a <const gomp_metadirective *> (gs),
+ spc, flags);
+ break;
+
case GIMPLE_CATCH:
dump_gimple_catch (buffer, as_a <const gcatch *> (gs), spc, flags);
break;
@@ -151,6 +151,7 @@ input_gimple_stmt (class lto_input_block *ib, class data_in *data_in,
case GIMPLE_COND:
case GIMPLE_GOTO:
case GIMPLE_DEBUG:
+ case GIMPLE_OMP_METADIRECTIVE:
for (i = 0; i < num_ops; i++)
{
tree *opp, op = stream_read_tree (ib, data_in);
@@ -188,6 +189,15 @@ input_gimple_stmt (class lto_input_block *ib, class data_in *data_in,
else
gimple_call_set_fntype (call_stmt, stream_read_tree (ib, data_in));
}
+ if (gomp_metadirective *metadirective_stmt
+ = dyn_cast <gomp_metadirective*> (stmt))
+ {
+ gimple_alloc_omp_metadirective (metadirective_stmt);
+ for (i = 0; i < num_ops; i++)
+ gimple_omp_metadirective_set_label (metadirective_stmt, i,
+ stream_read_tree (ib,
+ data_in));
+ }
break;
case GIMPLE_NOP:
@@ -127,6 +127,7 @@ output_gimple_stmt (struct output_block *ob, struct function *fn, gimple *stmt)
case GIMPLE_COND:
case GIMPLE_GOTO:
case GIMPLE_DEBUG:
+ case GIMPLE_OMP_METADIRECTIVE:
for (i = 0; i < gimple_num_ops (stmt); i++)
{
tree op = gimple_op (stmt, i);
@@ -169,6 +170,11 @@ output_gimple_stmt (struct output_block *ob, struct function *fn, gimple *stmt)
else
stream_write_tree (ob, gimple_call_fntype (stmt), true);
}
+ if (gimple_code (stmt) == GIMPLE_OMP_METADIRECTIVE)
+ for (i = 0; i < gimple_num_ops (stmt); i++)
+ stream_write_tree (ob, gimple_omp_metadirective_label (stmt, i),
+ true);
+
break;
case GIMPLE_NOP:
@@ -501,6 +501,20 @@ walk_gimple_op (gimple *stmt, walk_tree_fn callback_op,
return ret;
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ {
+ gimple_seq variant_seq = gimple_omp_variants (stmt);
+ for (gimple_stmt_iterator gsi = gsi_start (variant_seq);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ ret = walk_gimple_op (gimple_omp_body (gsi_stmt (gsi)),
+ callback_op, wi);
+ if (ret)
+ return ret;
+ }
+ }
+ break;
+
case GIMPLE_TRANSACTION:
{
gtransaction *txn = as_a <gtransaction *> (stmt);
@@ -717,6 +731,20 @@ walk_gimple_stmt (gimple_stmt_iterator *gsi, walk_stmt_fn callback_stmt,
return wi->callback_result;
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ {
+ gimple_seq variant_seq = gimple_omp_variants (stmt);
+ for (gimple_stmt_iterator gsi = gsi_start (variant_seq);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ ret = walk_gimple_seq_mod (gimple_omp_body_ptr (gsi_stmt (gsi)),
+ callback_stmt, callback_op, wi);
+ if (ret)
+ return wi->callback_result;
+ }
+ }
+ break;
+
case GIMPLE_WITH_CLEANUP_EXPR:
ret = walk_gimple_seq_mod (gimple_wce_cleanup_ptr (stmt), callback_stmt,
callback_op, wi);
@@ -1312,6 +1312,41 @@ gimple_build_assume (tree guard, gimple_seq body)
return p;
}
+/* Allocate extra memory for a GIMPLE_OMP_METADIRECTIVE statement. */
+
+void
+gimple_alloc_omp_metadirective (gimple *g)
+{
+ gomp_metadirective *p = as_a <gomp_metadirective *> (g);
+
+ p->labels = ggc_cleared_vec_alloc<tree> (gimple_num_ops (p));
+}
+
+/* Build a GIMPLE_OMP_METADIRECTIVE statement. */
+
+gomp_metadirective *
+gimple_build_omp_metadirective (int num_variants)
+{
+ gomp_metadirective *p
+ = as_a <gomp_metadirective *> (gimple_alloc (GIMPLE_OMP_METADIRECTIVE,
+ num_variants));
+ gimple_alloc_omp_metadirective (p);
+ gimple_omp_metadirective_set_variants (p, NULL);
+
+ return p;
+}
+
+/* Build a GIMPLE_OMP_METADIRECTIVE_VARIANT statement. */
+
+gomp_variant *
+gimple_build_omp_variant (gimple_seq body)
+{
+ gomp_variant *variant = as_a <gomp_variant *>
+ (gimple_alloc (GIMPLE_OMP_METADIRECTIVE_VARIANT, 0));
+ gimple_omp_set_body (variant, body);
+ return variant;
+}
+
/* Build a GIMPLE_TRANSACTION statement. */
gtransaction *
@@ -398,6 +398,13 @@ DEFGSCODE(GIMPLE_OMP_TEAMS, "gimple_omp_teams", GSS_OMP_PARALLEL_LAYOUT)
CLAUSES is an OMP_CLAUSE chain holding the associated clauses. */
DEFGSCODE(GIMPLE_OMP_ORDERED, "gimple_omp_ordered", GSS_OMP_SINGLE_LAYOUT)
+/* GIMPLE_OMP_METADIRECTIVE represents #pragma omp metadirective. */
+DEFGSCODE(GIMPLE_OMP_METADIRECTIVE, "gimple_omp_metadirective",
+ GSS_OMP_METADIRECTIVE)
+
+DEFGSCODE(GIMPLE_OMP_METADIRECTIVE_VARIANT,
+ "gimple_omp_variant", GSS_OMP_METADIRECTIVE_VARIANT)
+
/* GIMPLE_PREDICT <PREDICT, OUTCOME> specifies a hint for branch prediction.
PREDICT is one of the predictors from predict.def.
@@ -840,6 +840,30 @@ struct GTY((tag("GSS_ASSUME")))
gimple_seq body;
};
+struct GTY((tag("GSS_OMP_METADIRECTIVE_VARIANT")))
+ gomp_variant : public gimple_statement_omp
+{
+ /* The body in the base class contains the directive for this variant. */
+
+ /* No extra fields; adds invariant:
+ stmt->code == GIMPLE_OMP_METADIRECTIVE_VARIANT. */};
+
+struct GTY((tag("GSS_OMP_METADIRECTIVE")))
+ gomp_metadirective : public gimple_statement_with_ops_base
+{
+ /* [ WORD 1-7 ] : base class */
+
+ /* [ WORD 8 ] : a list of bodies associated with the directive variants. */
+ gomp_variant *variants;
+
+ /* [ WORD 9 ] : label vector. */
+ tree * GTY((length ("%h.num_ops"))) labels;
+
+ /* [ WORD 10 ] : operand vector. Used to hold the selectors for the
+ directive variants. */
+ tree GTY((length ("%h.num_ops"))) op[1];
+};
+
/* GIMPLE_TRANSACTION. */
/* Bits to be stored in the GIMPLE_TRANSACTION subcode. */
@@ -1251,6 +1275,22 @@ is_a_helper <gomp_task *>::test (gimple *gs)
return gs->code == GIMPLE_OMP_TASK;
}
+template <>
+template <>
+inline bool
+is_a_helper <gomp_metadirective *>::test (gimple *gs)
+{
+ return gs->code == GIMPLE_OMP_METADIRECTIVE;
+}
+
+template <>
+template <>
+inline bool
+is_a_helper <gomp_variant *>::test (gimple *gs)
+{
+ return gs->code == GIMPLE_OMP_METADIRECTIVE_VARIANT;
+}
+
template <>
template <>
inline bool
@@ -1501,6 +1541,22 @@ is_a_helper <const gomp_task *>::test (const gimple *gs)
return gs->code == GIMPLE_OMP_TASK;
}
+template <>
+template <>
+inline bool
+is_a_helper <const gomp_metadirective *>::test (const gimple *gs)
+{
+ return gs->code == GIMPLE_OMP_METADIRECTIVE;
+}
+
+template <>
+template <>
+inline bool
+is_a_helper <const gomp_variant *>::test (const gimple *gs)
+{
+ return gs->code == GIMPLE_OMP_METADIRECTIVE_VARIANT;
+}
+
template <>
template <>
inline bool
@@ -1609,6 +1665,9 @@ gomp_teams *gimple_build_omp_teams (gimple_seq, tree);
gomp_atomic_load *gimple_build_omp_atomic_load (tree, tree,
enum omp_memory_order);
gomp_atomic_store *gimple_build_omp_atomic_store (tree, enum omp_memory_order);
+void gimple_alloc_omp_metadirective (gimple *g);
+gomp_metadirective *gimple_build_omp_metadirective (int num_variants);
+gomp_variant *gimple_build_omp_variant (gimple_seq body);
gimple *gimple_build_assume (tree, gimple_seq);
gtransaction *gimple_build_transaction (gimple_seq);
extern void gimple_seq_add_stmt (gimple_seq *, gimple *);
@@ -1890,6 +1949,7 @@ gimple_has_substatements (gimple *g)
case GIMPLE_OMP_TARGET:
case GIMPLE_OMP_TEAMS:
case GIMPLE_OMP_CRITICAL:
+ case GIMPLE_OMP_METADIRECTIVE:
case GIMPLE_WITH_CLEANUP_EXPR:
case GIMPLE_TRANSACTION:
return true;
@@ -2148,7 +2208,8 @@ gimple_init_singleton (gimple *g)
inline bool
gimple_has_ops (const gimple *g)
{
- return gimple_code (g) >= GIMPLE_COND && gimple_code (g) <= GIMPLE_RETURN;
+ return (gimple_code (g) >= GIMPLE_COND && gimple_code (g) <= GIMPLE_RETURN)
+ || gimple_code (g) == GIMPLE_OMP_METADIRECTIVE;
}
template <>
@@ -6630,6 +6691,42 @@ gimple_assume_body (const gimple *gs)
return assume_stmt->body;
}
+
+static inline tree
+gimple_omp_metadirective_label (const gimple *g, unsigned i)
+{
+ const gomp_metadirective *omp_metadirective
+ = as_a <const gomp_metadirective *> (g);
+ return omp_metadirective->labels[i];
+}
+
+
+static inline void
+gimple_omp_metadirective_set_label (gimple *g, unsigned i, tree label)
+{
+ gomp_metadirective *omp_metadirective = as_a <gomp_metadirective *> (g);
+ omp_metadirective->labels[i] = label;
+}
+
+
+static inline gomp_variant *
+gimple_omp_variants (const gimple *g)
+{
+ const gomp_metadirective *omp_metadirective
+ = as_a <const gomp_metadirective *> (g);
+ return omp_metadirective->variants;
+}
+
+
+static inline void
+gimple_omp_metadirective_set_variants (gimple *g, gimple *variants)
+{
+ gomp_metadirective *omp_metadirective = as_a <gomp_metadirective *> (g);
+ omp_metadirective->variants
+ = variants ? as_a <gomp_variant *> (variants) : NULL;
+}
+
+
/* Return a pointer to the body for the GIMPLE_TRANSACTION statement
TRANSACTION_STMT. */
@@ -6781,6 +6878,7 @@ gimple_return_set_retval (greturn *gs, tree retval)
case GIMPLE_OMP_RETURN: \
case GIMPLE_OMP_ATOMIC_LOAD: \
case GIMPLE_OMP_ATOMIC_STORE: \
+ case GIMPLE_OMP_METADIRECTIVE: \
case GIMPLE_OMP_CONTINUE
inline bool
@@ -6313,6 +6313,7 @@ is_gimple_stmt (tree t)
case OMP_TASKGROUP:
case OMP_ORDERED:
case OMP_CRITICAL:
+ case OMP_METADIRECTIVE:
case OMP_TASK:
case OMP_TARGET:
case OMP_TARGET_DATA:
@@ -17611,6 +17612,184 @@ gimplify_omp_ordered (tree expr, gimple_seq body)
return gimple_build_omp_ordered (body, OMP_ORDERED_CLAUSES (expr));
}
+/* Replace a metadirective with the candidate directive variants in
+ CANDIDATES. */
+
+static enum gimplify_status
+expand_omp_metadirective (vec<struct omp_variant> &candidates,
+ gimple_seq *pre_p)
+{
+ auto_vec<tree> selectors;
+ auto_vec<tree> directive_labels;
+ auto_vec<gimple_seq> directive_bodies;
+ tree body_label = NULL_TREE;
+ tree end_label = create_artificial_label (UNKNOWN_LOCATION);
+
+ /* Construct bodies for each candidate. */
+ for (unsigned i = 0; i < candidates.length(); i++)
+ {
+ struct omp_variant &candidate = candidates[i];
+ gimple_seq body = NULL;
+
+ selectors.safe_push (candidate.dynamic_selector);
+ directive_labels.safe_push (create_artificial_label (UNKNOWN_LOCATION));
+
+ gimplify_seq_add_stmt (&body,
+ gimple_build_label (directive_labels.last ()));
+ if (candidate.alternative != NULL_TREE)
+ gimplify_stmt (&candidate.alternative, &body);
+ if (candidate.body != NULL_TREE)
+ {
+ if (body_label != NULL_TREE)
+ gimplify_seq_add_stmt (&body, gimple_build_goto (body_label));
+ else
+ {
+ body_label = create_artificial_label (UNKNOWN_LOCATION);
+ gimplify_seq_add_stmt (&body, gimple_build_label (body_label));
+ gimplify_stmt (&candidate.body, &body);
+ }
+ }
+
+ directive_bodies.safe_push (body);
+ }
+
+ auto_vec<tree> cond_labels;
+
+ cond_labels.safe_push (NULL_TREE);
+ for (unsigned i = 1; i < candidates.length () - 1; i++)
+ cond_labels.safe_push (create_artificial_label (UNKNOWN_LOCATION));
+ if (candidates.length () > 1)
+ cond_labels.safe_push (directive_labels.last ());
+
+ /* Generate conditionals to test each dynamic selector in turn, executing
+ the directive candidate if successful. */
+ for (unsigned i = 0; i < candidates.length () - 1; i++)
+ {
+ if (i != 0)
+ gimplify_seq_add_stmt (pre_p, gimple_build_label (cond_labels [i]));
+
+ enum gimplify_status ret = gimplify_expr (&selectors[i], pre_p, NULL,
+ is_gimple_val, fb_rvalue);
+ if (ret == GS_ERROR || ret == GS_UNHANDLED)
+ return ret;
+
+ gcond *cond_stmt
+ = gimple_build_cond_from_tree (selectors[i], directive_labels[i],
+ cond_labels[i + 1]);
+
+ gimplify_seq_add_stmt (pre_p, cond_stmt);
+ gimplify_seq_add_seq (pre_p, directive_bodies[i]);
+ gimplify_seq_add_stmt (pre_p, gimple_build_goto (end_label));
+ }
+
+ gimplify_seq_add_seq (pre_p, directive_bodies.last ());
+ gimplify_seq_add_stmt (pre_p, gimple_build_label (end_label));
+
+ return GS_ALL_DONE;
+}
+
+/* Gimplify an OMP_METADIRECTIVE construct. EXPR is the tree version.
+ The metadirective will be resolved at this point if possible. */
+
+static enum gimplify_status
+gimplify_omp_metadirective (tree *expr_p, gimple_seq *pre_p, gimple_seq *,
+ bool (*) (tree), fallback_t)
+{
+ auto_vec<tree> selectors;
+
+ /* Mark offloadable functions containing metadirectives that specify
+ a 'construct' selector with a 'target' constructor. */
+ if (offloading_function_p (current_function_decl))
+ {
+ for (tree variant = OMP_METADIRECTIVE_VARIANTS (*expr_p);
+ variant != NULL_TREE; variant = TREE_CHAIN (variant))
+ {
+ tree selector = OMP_METADIRECTIVE_VARIANT_SELECTOR (variant);
+
+ if (omp_get_context_selector (selector, OMP_TRAIT_SET_CONSTRUCT,
+ OMP_TRAIT_CONSTRUCT_TARGET))
+ {
+ tree id = get_identifier ("omp metadirective construct target");
+
+ DECL_ATTRIBUTES (current_function_decl)
+ = tree_cons (id, NULL_TREE,
+ DECL_ATTRIBUTES (current_function_decl));
+ break;
+ }
+ }
+ }
+
+ /* Try to resolve the metadirective. */
+ vec<struct omp_variant> candidates
+ = omp_early_resolve_metadirective (*expr_p);
+ if (!candidates.is_empty ())
+ return expand_omp_metadirective (candidates, pre_p);
+
+ /* The metadirective cannot be resolved yet. */
+
+ gomp_variant *first_variant = NULL;
+ gomp_variant *prev_variant = NULL;
+ gimple_seq standalone_body = NULL;
+ tree body_label = NULL;
+ tree end_label = create_artificial_label (UNKNOWN_LOCATION);
+
+ for (tree variant = OMP_METADIRECTIVE_VARIANTS (*expr_p); variant != NULL_TREE;
+ variant = TREE_CHAIN (variant))
+ {
+ tree selector = OMP_METADIRECTIVE_VARIANT_SELECTOR (variant);
+ tree directive = OMP_METADIRECTIVE_VARIANT_DIRECTIVE (variant);
+ tree body = OMP_METADIRECTIVE_VARIANT_BODY (variant);
+
+ selectors.safe_push (selector);
+ gomp_variant *omp_variant
+ = gimple_build_omp_variant (NULL);
+ gimple_seq *directive_p = gimple_omp_body_ptr (omp_variant);
+
+ gimplify_stmt (&directive, directive_p);
+ if (body != NULL_TREE)
+ {
+ if (standalone_body == NULL)
+ {
+ gimplify_stmt (&body, &standalone_body);
+ body_label = create_artificial_label (UNKNOWN_LOCATION);
+ }
+ gimplify_seq_add_stmt (directive_p, gimple_build_goto (body_label));
+ }
+ else
+ gimplify_seq_add_stmt (directive_p, gimple_build_goto (end_label));
+
+ if (!first_variant)
+ first_variant = omp_variant;
+ if (prev_variant)
+ {
+ prev_variant->next = omp_variant;
+ omp_variant->prev = prev_variant;
+ }
+ prev_variant = omp_variant;
+ }
+
+ gomp_metadirective *stmt
+ = gimple_build_omp_metadirective (selectors.length ());
+ gimple_omp_metadirective_set_variants (stmt, first_variant);
+
+ tree selector;
+ unsigned int i;
+ FOR_EACH_VEC_ELT (selectors, i, selector)
+ gimple_set_op (stmt, i, selector);
+
+ gimplify_seq_add_stmt (pre_p, stmt);
+ if (standalone_body)
+ {
+ gimplify_seq_add_stmt (pre_p, gimple_build_label (body_label));
+ gimplify_seq_add_stmt (pre_p, standalone_body);
+ }
+ gimplify_seq_add_stmt (pre_p, gimple_build_label (end_label));
+
+ cgraph_node::get (cfun->decl)->has_metadirectives = 1;
+
+ return GS_ALL_DONE;
+}
+
/* Convert the GENERIC expression tree *EXPR_P to GIMPLE. If the
expression produces a value to be used as an operand inside a GIMPLE
statement, the value will be stored back in *EXPR_P. This value will
@@ -18537,6 +18716,11 @@ gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
ret = gimplify_omp_atomic (expr_p, pre_p);
break;
+ case OMP_METADIRECTIVE:
+ ret = gimplify_omp_metadirective (expr_p, pre_p, post_p,
+ gimple_test_f, fallback);
+ break;
+
case TRANSACTION_EXPR:
ret = gimplify_transaction (expr_p, pre_p);
break;
@@ -51,4 +51,6 @@ DEFGSSTRUCT(GSS_OMP_CONTINUE, gomp_continue, false)
DEFGSSTRUCT(GSS_OMP_ATOMIC_LOAD, gomp_atomic_load, false)
DEFGSSTRUCT(GSS_OMP_ATOMIC_STORE_LAYOUT, gomp_atomic_store, false)
DEFGSSTRUCT(GSS_ASSUME, gimple_statement_assume, false)
+DEFGSSTRUCT(GSS_OMP_METADIRECTIVE, gomp_metadirective, true)
+DEFGSSTRUCT(GSS_OMP_METADIRECTIVE_VARIANT, gomp_variant, false)
DEFGSSTRUCT(GSS_TRANSACTION, gtransaction, false)
@@ -551,6 +551,7 @@ lto_output_node (struct lto_simple_output_block *ob, struct cgraph_node *node,
bp_pack_value (&bp, node->parallelized_function, 1);
bp_pack_value (&bp, node->declare_variant_alt, 1);
bp_pack_value (&bp, node->calls_declare_variant_alt, 1);
+ bp_pack_value (&bp, node->has_metadirectives, 1);
/* Stream thunk info always because we use it in
ipa_polymorphic_call_context::ipa_polymorphic_call_context
@@ -1252,6 +1253,7 @@ input_overwrite_node (struct lto_file_decl_data *file_data,
node->parallelized_function = bp_unpack_value (bp, 1);
node->declare_variant_alt = bp_unpack_value (bp, 1);
node->calls_declare_variant_alt = bp_unpack_value (bp, 1);
+ node->has_metadirectives = bp_unpack_value (bp, 1);
*has_thunk_info = bp_unpack_value (bp, 1);
node->resolution = bp_unpack_enum (bp, ld_plugin_symbol_resolution,
LDPR_NUM_KNOWN);
@@ -10016,6 +10016,8 @@ expand_omp_target (struct omp_region *region)
child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
cgraph_node *node = cgraph_node::get_create (child_fn);
node->parallelized_function = 1;
+ node->has_metadirectives
+ |= cgraph_node::get (cfun->decl)->has_metadirectives;
cgraph_node::add_new_function (child_fn, true);
/* Add the new function to the offload table. */
@@ -10752,6 +10754,10 @@ build_omp_regions_1 (basic_block bb, struct omp_region *parent,
/* GIMPLE_OMP_SECTIONS_SWITCH is part of
GIMPLE_OMP_SECTIONS, and we do nothing for it. */
}
+ else if (code == GIMPLE_OMP_METADIRECTIVE)
+ {
+ /* Do nothing for metadirectives. */
+ }
else
{
region = new_omp_region (bb, code, parent);
@@ -11137,6 +11143,30 @@ omp_make_gimple_edges (basic_block bb, struct omp_region **region,
}
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ /* Create an edge to the beginning of the body of each candidate
+ directive. */
+ {
+ gimple *stmt = last_nondebug_stmt (bb);
+ unsigned i;
+ bool seen_default = false;
+
+ for (i = 0; i < gimple_num_ops (stmt); i++)
+ {
+ tree dest = gimple_omp_metadirective_label (stmt, i);
+ basic_block dest_bb = label_to_block (cfun, dest);
+ make_edge (bb, dest_bb, 0);
+
+ if (gimple_op (stmt, i) == NULL_TREE)
+ seen_default = true;
+ }
+
+ gcc_assert (seen_default);
+
+ fallthru = false;
+ }
+ break;
+
default:
gcc_unreachable ();
}
@@ -3187,6 +3187,28 @@ omp_early_resolve_metadirective (tree metadirective)
return omp_get_dynamic_candidates (candidates, true);
}
+/* Return a vector of dynamic replacement candidates for the metadirective
+ Gimple statement in GS. Return an empty vector if the metadirective
+ cannot be resolved. */
+
+vec<struct omp_variant>
+omp_late_resolve_metadirective (gimple *gs)
+{
+ auto_vec <struct omp_variant> variants;
+
+ for (unsigned i = 0; i < gimple_num_ops (gs); i++)
+ {
+ struct omp_variant variant;
+
+ variant.selector = gimple_op (gs, i);
+ variant.alternative = gimple_omp_metadirective_label (gs, i);
+
+ variants.safe_push (variant);
+ }
+
+ return omp_get_dynamic_candidates (variants, false);
+}
+
/* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
macro on gomp-constants.h. We do not check for overflow. */
@@ -202,6 +202,7 @@ extern tree omp_get_context_selector (tree, enum omp_tss_code,
extern tree omp_get_context_selector_list (tree, enum omp_tss_code);
extern tree omp_resolve_declare_variant (tree);
extern vec<struct omp_variant> omp_early_resolve_metadirective (tree);
+extern vec<struct omp_variant> omp_late_resolve_metadirective (gimple *);
extern tree oacc_launch_pack (unsigned code, tree device, unsigned op);
extern tree oacc_replace_fn_attrib_attr (tree attribs, tree dims);
extern void oacc_replace_fn_attrib (tree fn, tree dims);
@@ -183,6 +183,10 @@ struct omp_context
/* Candidates for adjusting OpenACC privatization level. */
vec<tree> oacc_privatization_candidates;
+
+ /* Only used for omp metadirectives. Links to the next shallow
+ clone of this context. */
+ struct omp_context *next_clone;
};
static splay_tree all_contexts;
@@ -974,6 +978,7 @@ new_omp_context (gimple *stmt, omp_context *outer_ctx)
splay_tree_insert (all_contexts, (splay_tree_key) stmt,
(splay_tree_value) ctx);
ctx->stmt = stmt;
+ ctx->next_clone = NULL;
if (outer_ctx)
{
@@ -1003,6 +1008,18 @@ new_omp_context (gimple *stmt, omp_context *outer_ctx)
return ctx;
}
+static omp_context *
+clone_omp_context (omp_context *ctx)
+{
+ omp_context *clone_ctx = XCNEW (omp_context);
+
+ memcpy (clone_ctx, ctx, sizeof (omp_context));
+ ctx->next_clone = clone_ctx;
+ clone_ctx->next_clone = NULL;
+
+ return clone_ctx;
+}
+
static gimple_seq maybe_catch_exception (gimple_seq);
/* Finalize task copyfn. */
@@ -1049,6 +1066,15 @@ delete_omp_context (splay_tree_value value)
{
omp_context *ctx = (omp_context *) value;
+ /* Delete clones. */
+ omp_context *clone = ctx->next_clone;
+ while (clone)
+ {
+ omp_context *next_clone = clone->next_clone;
+ XDELETE (clone);
+ clone = next_clone;
+ }
+
delete ctx->cb.decl_map;
if (ctx->field_map)
@@ -2093,6 +2119,9 @@ create_omp_child_function (omp_context *ctx, bool task_copy)
DECL_FUNCTION_VERSIONED (decl)
= DECL_FUNCTION_VERSIONED (current_function_decl);
+ if (cgraph_node::get (cfun->decl)->has_metadirectives)
+ cgraph_node::get_create (decl)->has_metadirectives = 1;
+
if (omp_maybe_offloaded_ctx (ctx))
{
cgraph_node::get_create (decl)->offloadable = 1;
@@ -3182,6 +3211,22 @@ scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
ctx->record_type = ctx->receiver_decl = NULL;
}
+/* Scan an OpenMP metadirective. */
+
+static void
+scan_omp_metadirective (gomp_metadirective *stmt, omp_context *outer_ctx)
+{
+ gimple_seq variant_seq = gimple_omp_variants (stmt);
+ for (gimple_stmt_iterator gsi = gsi_start (variant_seq);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple_seq *directive_p = gimple_omp_body_ptr (gsi_stmt (gsi));
+ omp_context *ctx = outer_ctx ? clone_omp_context (outer_ctx) : NULL;
+
+ scan_omp (directive_p, ctx);
+ }
+}
+
/* Check nesting restrictions. */
static bool
check_omp_nesting_restrictions (gimple *stmt, omp_context *ctx)
@@ -4245,6 +4290,10 @@ scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ scan_omp_metadirective (as_a <gomp_metadirective *> (stmt), ctx);
+ break;
+
case GIMPLE_BIND:
{
tree var;
@@ -10702,6 +10751,19 @@ oacc_privatization_scan_decl_chain (omp_context *ctx, tree decls)
}
}
+static void
+lower_omp_metadirective (gimple_stmt_iterator *gsi_p, omp_context *ctx)
+{
+ gimple *stmt = gsi_stmt (*gsi_p);
+ gimple_seq variant_seq = gimple_omp_variants (stmt);
+ for (gimple_stmt_iterator gsi = gsi_start (variant_seq);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple_seq *directive_p = gimple_omp_body_ptr (gsi_stmt (gsi));
+ lower_omp (directive_p, ctx);
+ }
+}
+
/* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
static tree
@@ -14458,10 +14520,31 @@ lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
else
lower_omp_teams (gsi_p, ctx);
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ lower_omp_metadirective (gsi_p, ctx);
+ break;
case GIMPLE_CALL:
tree fndecl;
call_stmt = as_a <gcall *> (stmt);
fndecl = gimple_call_fndecl (call_stmt);
+ if (fndecl
+ && lookup_attribute ("omp metadirective construct target",
+ DECL_ATTRIBUTES (fndecl)))
+ {
+ bool in_target_ctx = false;
+
+ for (omp_context *up = ctx; up; up = up->outer)
+ if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
+ {
+ in_target_ctx = true;
+ break;
+ }
+ if (!ctx || !in_target_ctx)
+ warning_at (gimple_location (stmt), 0,
+ "direct calls to an offloadable function containing "
+ "metadirectives with a %<construct={target}%> "
+ "selector may produce unexpected results");
+ }
if (fndecl
&& fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
switch (DECL_FUNCTION_CODE (fndecl))
@@ -55,6 +55,8 @@ along with GCC; see the file COPYING3. If not see
#include "context.h"
#include "convert.h"
#include "opts.h"
+#include "cfganal.h"
+#include "cfghooks.h"
/* Describe the OpenACC looping structure of a function. The entire
function is held in a 'NULL' loop. */
@@ -1954,6 +1956,92 @@ is_sync_builtin_call (gcall *call)
return false;
}
+/* Resolve an OpenMP metadirective in the function FUN, in the basic block
+ BB. The metadirective should be the last statement in BB. */
+
+static void
+omp_expand_metadirective (function *fun, basic_block bb)
+{
+ gimple *stmt = last_nondebug_stmt (bb);
+ vec<struct omp_variant> candidates
+ = omp_late_resolve_metadirective (stmt);
+
+ /* This is the last chance for the metadirective to be resolved. */
+ gcc_assert (!candidates.is_empty ());
+
+ auto_vec<tree> labels;
+
+ for (unsigned int i = 0; i < candidates.length (); i++)
+ labels.safe_push (candidates[i].alternative);
+
+ /* Delete BBs for all variants not in the candidate list. */
+ for (unsigned i = 0; i < gimple_num_ops (stmt); i++)
+ {
+ tree label = gimple_omp_metadirective_label (stmt, i);
+ if (!labels.contains (label))
+ {
+ edge e = find_edge (bb, label_to_block (fun, label));
+ remove_edge_and_dominated_blocks (e);
+ labels.safe_push (label);
+ }
+ }
+
+ /* Remove the metadirective statement. */
+ gimple_stmt_iterator gsi = gsi_last_bb (bb);
+ gsi_remove (&gsi, true);
+
+ if (candidates.length () == 1)
+ {
+ /* Special case if there is only one selector - there should be one
+ remaining edge from BB to the selected variant. */
+ edge e = find_edge (bb, label_to_block (fun,
+ candidates.last ().alternative));
+ e->flags |= EDGE_FALLTHRU;
+
+ return;
+ }
+
+ basic_block cur_bb = bb;
+
+ /* For each candidate, create a conditional that checks the dynamic
+ condition, branching to the candidate directive if true, to the
+ next candidate check if false. */
+ for (unsigned i = 0; i < candidates.length () - 1; i++)
+ {
+ basic_block next_bb = NULL;
+ gcond *cond_stmt
+ = gimple_build_cond_from_tree (candidates[i].dynamic_selector,
+ NULL_TREE, NULL_TREE);
+ gsi = gsi_last_bb (cur_bb);
+ gsi_insert_seq_after (&gsi, cond_stmt, GSI_NEW_STMT);
+
+ if (i < candidates.length () - 2)
+ {
+ edge e_false = split_block (cur_bb, cond_stmt);
+ e_false->flags &= ~EDGE_FALLTHRU;
+ e_false->flags |= EDGE_FALSE_VALUE;
+ e_false->probability = profile_probability::uninitialized ();
+
+ next_bb = e_false->dest;
+ }
+
+ /* Redirect the source of the edge from BB to the candidate directive
+ to the conditional. Reusing the edge avoids disturbing phi nodes in
+ the destination BB. */
+ edge e = find_edge (bb, label_to_block (fun, candidates[i].alternative));
+ redirect_edge_pred (e, cur_bb);
+ e->flags |= EDGE_TRUE_VALUE;
+
+ if (next_bb)
+ cur_bb = next_bb;
+ }
+
+ /* The last of the candidates is always static. */
+ edge e = find_edge (cur_bb, label_to_block (fun,
+ candidates.last ().alternative));
+ e->flags |= EDGE_FALSE_VALUE;
+}
+
/* Main entry point for oacc transformations which run on the device
compiler after LTO, so we know what the target device is at this
point (including the host fallback). */
@@ -2632,6 +2720,7 @@ execute_omp_device_lower ()
gimple_stmt_iterator gsi;
bool calls_declare_variant_alt
= cgraph_node::get (cfun->decl)->calls_declare_variant_alt;
+ auto_vec<basic_block> metadirective_bbs;
#ifdef ACCEL_COMPILER
bool omp_redirect_indirect_calls = vec_safe_length (offload_ind_funcs) > 0;
tree map_ptr_fn
@@ -2641,6 +2730,8 @@ execute_omp_device_lower ()
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
+ if (is_a<gomp_metadirective *> (stmt))
+ metadirective_bbs.safe_push (bb);
if (!is_gimple_call (stmt))
continue;
if (!gimple_call_internal_p (stmt))
@@ -2790,6 +2881,16 @@ execute_omp_device_lower ()
}
if (vf != 1)
cfun->has_force_vectorize_loops = false;
+ if (!metadirective_bbs.is_empty ())
+ {
+ calculate_dominance_info (CDI_DOMINATORS);
+
+ for (unsigned i = 0; i < metadirective_bbs.length (); i++)
+ omp_expand_metadirective (cfun, metadirective_bbs[i]);
+
+ free_dominance_info (cfun, CDI_DOMINATORS);
+ mark_virtual_operands_for_renaming (cfun);
+ }
return 0;
}
@@ -2818,6 +2919,7 @@ public:
/* opt_pass methods: */
bool gate (function *fun) final override
{
+ cgraph_node *node = cgraph_node::get (fun->decl);
#ifdef ACCEL_COMPILER
bool offload_ind_funcs_p = vec_safe_length (offload_ind_funcs) > 0;
#else
@@ -2825,7 +2927,8 @@ public:
#endif
return (!(fun->curr_properties & PROP_gimple_lomp_dev)
|| (flag_openmp
- && (cgraph_node::get (fun->decl)->calls_declare_variant_alt
+ && (node->calls_declare_variant_alt
+ || node->has_metadirectives
|| offload_ind_funcs_p)));
}
unsigned int execute (function *) final override
@@ -690,6 +690,7 @@ simd_clone_create (struct cgraph_node *old_node, bool force_local)
new_node->externally_visible = old_node->externally_visible;
new_node->calls_declare_variant_alt
= old_node->calls_declare_variant_alt;
+ new_node->has_metadirectives = old_node->has_metadirectives;
}
/* Mark clones with internal linkage as gc'able, so they will not be
@@ -1752,6 +1752,18 @@ cleanup_dead_labels (void)
}
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ {
+ for (unsigned i = 0; i < gimple_num_ops (stmt); i++)
+ {
+ label = gimple_omp_metadirective_label (stmt, i);
+ new_label = main_block_label (label, label_for_bb);
+ if (new_label != label)
+ gimple_omp_metadirective_set_label (stmt, i, new_label);
+ }
+ }
+ break;
+
default:
break;
}
@@ -6315,6 +6327,18 @@ gimple_redirect_edge_and_branch (edge e, basic_block dest)
gimple_block_label (dest));
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ {
+ for (unsigned i = 0; i < gimple_num_ops (stmt); i++)
+ {
+ tree label = gimple_omp_metadirective_label (stmt, i);
+ if (label_to_block (cfun, label) == e->dest)
+ gimple_omp_metadirective_set_label (stmt, i,
+ gimple_block_label (dest));
+ }
+ }
+ break;
+
default:
/* Otherwise it must be a fallthru edge, and we don't need to
do anything besides redirecting it. */
@@ -1672,6 +1672,36 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
(s1, gimple_omp_masked_clauses (stmt));
break;
+ case GIMPLE_OMP_METADIRECTIVE:
+ copy = gimple_build_omp_metadirective (gimple_num_ops (stmt));
+ {
+ gimple *first_variant = NULL;
+ gimple **prev_next = &first_variant;
+ gimple_seq variant_seq = gimple_omp_variants (stmt);
+ for (gimple_stmt_iterator gsi = gsi_start (variant_seq);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ s1 = remap_gimple_seq (gimple_omp_body (gsi_stmt (gsi)), id);
+ gimple *new_variant
+ = gimple_build_omp_variant (s1);
+
+ *prev_next = new_variant;
+ prev_next = &new_variant->next;
+ }
+ gimple_omp_metadirective_set_variants (copy, first_variant);
+ }
+
+ memset (&wi, 0, sizeof (wi));
+ wi.info = id;
+ for (unsigned i = 0; i < gimple_num_ops (stmt); i++)
+ {
+ tree label = gimple_omp_metadirective_label (stmt, i);
+ walk_tree (&label, remap_gimple_op_r, &wi, NULL);
+ gimple_omp_metadirective_set_label (copy, i, label);
+ gimple_set_op (copy, i, gimple_op (stmt, i));
+ }
+ break;
+
case GIMPLE_OMP_SCOPE:
s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
copy = gimple_build_omp_scope
@@ -4607,6 +4637,13 @@ estimate_num_insns (gimple *stmt, eni_weights *weights)
return (weights->omp_cost
+ estimate_num_insns_seq (gimple_omp_body (stmt), weights));
+ case GIMPLE_OMP_METADIRECTIVE:
+ /* The actual instruction will disappear eventually, so metadirective
+ statements have zero additional cost (if only static selectors
+ are used). */
+ /* TODO: Estimate the cost of evaluating dynamic selectors */
+ return 0;
+
case GIMPLE_TRANSACTION:
return (weights->tm_cost
+ estimate_num_insns_seq (gimple_transaction_body (
@@ -5021,6 +5058,7 @@ expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id,
dst_cfun->calls_eh_return |= id->src_cfun->calls_eh_return;
id->dst_node->calls_declare_variant_alt
|= id->src_node->calls_declare_variant_alt;
+ id->dst_node->has_metadirectives |= id->src_node->has_metadirectives;
gcc_assert (!id->src_cfun->after_inlining);
@@ -6276,6 +6314,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
new_entry ? new_entry->count : old_entry_block->count);
new_version_node->calls_declare_variant_alt
= old_version_node->calls_declare_variant_alt;
+ new_version_node->has_metadirectives = old_version_node->has_metadirectives;
if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
= id.src_cfun->gimple_df->ipa_pta;
@@ -28,6 +28,7 @@ along with GCC; see the file COPYING3. If not see
#include "gimple-pretty-print.h"
#include "diagnostic-core.h"
#include "stmt.h"
+#include "omp-general.h"
#include "print-tree.h"
#include "dumpfile.h"
@@ -972,6 +973,22 @@ operands_scanner::parse_ssa_operands ()
append_vuse (gimple_vop (fn));
goto do_default;
+ case GIMPLE_OMP_METADIRECTIVE:
+ n = gimple_num_ops (stmt);
+ for (i = start; i < n; i++)
+ for (tree tss = gimple_op (stmt, i);
+ tss != NULL; tss = TREE_CHAIN (tss))
+ if (OMP_TSS_CODE (tss) == OMP_TRAIT_SET_USER
+ || OMP_TSS_CODE (tss) == OMP_TRAIT_SET_TARGET_DEVICE)
+ for (tree ts = OMP_TSS_TRAIT_SELECTORS (tss);
+ ts != NULL; ts = TREE_CHAIN (ts))
+ if (OMP_TS_CODE (ts) == OMP_TRAIT_USER_CONDITION
+ || OMP_TS_CODE (ts) == OMP_TRAIT_DEVICE_NUM)
+ for (tree tp = OMP_TS_PROPERTIES (ts);
+ tp != NULL; tp = TREE_CHAIN (tp))
+ get_expr_operands (&OMP_TP_VALUE (tp), opf_use);
+ break;
+
case GIMPLE_CALL:
/* Add call-clobbered operands, if needed. */
maybe_add_call_vops (as_a <gcall *> (stmt));