#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "errors.h"
#include "ggc.h"
#include "tree.h"
#include "target.h"
#include "rtl.h"
#include "basic-block.h"
#include "diagnostic.h"
#include "tree-flow.h"
#include "tree-dump.h"
#include "timevar.h"
#include "cfgloop.h"
#include "cfglayout.h"
#include "expr.h"
#include "optabs.h"
#include "toplev.h"
#include "tree-chrec.h"
#include "tree-data-ref.h"
#include "tree-scalar-evolution.h"
#include "input.h"
#include "tree-vectorizer.h"
#include "tree-pass.h"
static struct loop *slpeel_tree_duplicate_loop_to_edge_cfg
(struct loop *, struct loops *, edge);
static void slpeel_update_phis_for_duplicate_loop
(struct loop *, struct loop *, bool after);
static void slpeel_update_phi_nodes_for_guard (edge, struct loop *, bool, bool);
static edge slpeel_add_loop_guard (basic_block, tree, basic_block, basic_block);
static void allocate_new_names (bitmap);
static void rename_use_op (use_operand_p);
static void rename_def_op (def_operand_p, tree);
static void rename_variables_in_bb (basic_block);
static void free_new_names (bitmap);
static void rename_variables_in_loop (struct loop *);
static void vect_set_dump_settings (void);
static bool need_imm_uses_for (tree);
FILE *vect_dump;
enum verbosity_levels vect_verbosity_level = MAX_VERBOSITY_LEVEL;
unsigned int loops_num;
static LOC vect_loop_location;
static void
allocate_new_names (bitmap definitions)
{
unsigned ver;
bitmap_iterator bi;
EXECUTE_IF_SET_IN_BITMAP (definitions, 0, ver, bi)
{
tree def = ssa_name (ver);
tree *new_name_ptr = xmalloc (sizeof (tree));
bool abnormal = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def);
*new_name_ptr = duplicate_ssa_name (def, SSA_NAME_DEF_STMT (def));
SSA_NAME_OCCURS_IN_ABNORMAL_PHI (*new_name_ptr) = abnormal;
SSA_NAME_AUX (def) = new_name_ptr;
}
}
static void
rename_use_op (use_operand_p op_p)
{
tree *new_name_ptr;
if (TREE_CODE (USE_FROM_PTR (op_p)) != SSA_NAME)
return;
new_name_ptr = SSA_NAME_AUX (USE_FROM_PTR (op_p));
if (!new_name_ptr)
return;
SET_USE (op_p, *new_name_ptr);
}
static void
rename_def_op (def_operand_p op_p, tree stmt)
{
tree *new_name_ptr;
if (TREE_CODE (DEF_FROM_PTR (op_p)) != SSA_NAME)
return;
new_name_ptr = SSA_NAME_AUX (DEF_FROM_PTR (op_p));
if (!new_name_ptr)
return;
SET_DEF (op_p, *new_name_ptr);
SSA_NAME_DEF_STMT (DEF_FROM_PTR (op_p)) = stmt;
}
static void
rename_variables_in_bb (basic_block bb)
{
tree phi;
block_stmt_iterator bsi;
tree stmt;
stmt_ann_t ann;
use_optype uses;
vuse_optype vuses;
def_optype defs;
v_may_def_optype v_may_defs;
v_must_def_optype v_must_defs;
unsigned i;
edge e;
edge_iterator ei;
struct loop *loop = bb->loop_father;
for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
rename_def_op (PHI_RESULT_PTR (phi), phi);
for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))
{
stmt = bsi_stmt (bsi);
get_stmt_operands (stmt);
ann = stmt_ann (stmt);
uses = USE_OPS (ann);
for (i = 0; i < NUM_USES (uses); i++)
rename_use_op (USE_OP_PTR (uses, i));
defs = DEF_OPS (ann);
for (i = 0; i < NUM_DEFS (defs); i++)
rename_def_op (DEF_OP_PTR (defs, i), stmt);
vuses = VUSE_OPS (ann);
for (i = 0; i < NUM_VUSES (vuses); i++)
rename_use_op (VUSE_OP_PTR (vuses, i));
v_may_defs = V_MAY_DEF_OPS (ann);
for (i = 0; i < NUM_V_MAY_DEFS (v_may_defs); i++)
{
rename_use_op (V_MAY_DEF_OP_PTR (v_may_defs, i));
rename_def_op (V_MAY_DEF_RESULT_PTR (v_may_defs, i), stmt);
}
v_must_defs = V_MUST_DEF_OPS (ann);
for (i = 0; i < NUM_V_MUST_DEFS (v_must_defs); i++)
{
rename_use_op (V_MUST_DEF_KILL_PTR (v_must_defs, i));
rename_def_op (V_MUST_DEF_RESULT_PTR (v_must_defs, i), stmt);
}
}
FOR_EACH_EDGE (e, ei, bb->succs)
{
if (!flow_bb_inside_loop_p (loop, e->dest))
continue;
for (phi = phi_nodes (e->dest); phi; phi = PHI_CHAIN (phi))
rename_use_op (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e));
}
}
static void
free_new_names (bitmap definitions)
{
unsigned ver;
bitmap_iterator bi;
EXECUTE_IF_SET_IN_BITMAP (definitions, 0, ver, bi)
{
tree def = ssa_name (ver);
if (SSA_NAME_AUX (def))
{
free (SSA_NAME_AUX (def));
SSA_NAME_AUX (def) = NULL;
}
}
}
static void
rename_variables_in_loop (struct loop *loop)
{
unsigned i;
basic_block *bbs;
bbs = get_loop_body (loop);
for (i = 0; i < loop->num_nodes; i++)
rename_variables_in_bb (bbs[i]);
free (bbs);
}
static void
slpeel_update_phis_for_duplicate_loop (struct loop *orig_loop,
struct loop *new_loop, bool after)
{
tree *new_name_ptr, new_ssa_name;
tree phi_new, phi_orig;
tree def;
edge orig_loop_latch = loop_latch_edge (orig_loop);
edge orig_entry_e = loop_preheader_edge (orig_loop);
edge new_loop_exit_e = new_loop->exit_edges[0];
edge new_loop_entry_e = loop_preheader_edge (new_loop);
edge entry_arg_e = (after ? orig_loop_latch : orig_entry_e);
for (phi_new = phi_nodes (new_loop->header),
phi_orig = phi_nodes (orig_loop->header);
phi_new && phi_orig;
phi_new = PHI_CHAIN (phi_new), phi_orig = PHI_CHAIN (phi_orig))
{
def = PHI_ARG_DEF_FROM_EDGE (phi_orig, entry_arg_e);
add_phi_arg (phi_new, def, new_loop_entry_e);
def = PHI_ARG_DEF_FROM_EDGE (phi_orig, orig_loop_latch);
if (TREE_CODE (def) != SSA_NAME)
continue;
new_name_ptr = SSA_NAME_AUX (def);
if (!new_name_ptr)
continue;
new_ssa_name = *new_name_ptr;
add_phi_arg (phi_new, new_ssa_name, loop_latch_edge (new_loop));
if (!after)
{
gcc_assert (new_loop_exit_e == orig_entry_e);
SET_PHI_ARG_DEF (phi_orig,
new_loop_exit_e->dest_idx,
new_ssa_name);
}
}
}
static void
slpeel_update_phi_nodes_for_guard (edge guard_edge,
struct loop *loop,
bool entry_phis,
bool is_new_loop)
{
tree orig_phi, new_phi, update_phi;
tree guard_arg, loop_arg;
basic_block new_merge_bb = guard_edge->dest;
edge e = EDGE_SUCC (new_merge_bb, 0);
basic_block update_bb = e->dest;
basic_block orig_bb = (entry_phis ? loop->header : update_bb);
for (orig_phi = phi_nodes (orig_bb), update_phi = phi_nodes (update_bb);
orig_phi && update_phi;
orig_phi = PHI_CHAIN (orig_phi), update_phi = PHI_CHAIN (update_phi))
{
new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),
new_merge_bb);
if (entry_phis)
{
loop_arg = PHI_ARG_DEF_FROM_EDGE (orig_phi,
EDGE_SUCC (loop->latch, 0));
guard_arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, loop->entry_edges[0]);
}
else
{
tree orig_def = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
tree *new_name_ptr = SSA_NAME_AUX (orig_def);
tree new_name;
if (new_name_ptr)
new_name = *new_name_ptr;
else
new_name = orig_def;
if (is_new_loop)
{
guard_arg = orig_def;
loop_arg = new_name;
}
else
{
guard_arg = new_name;
loop_arg = orig_def;
}
}
add_phi_arg (new_phi, loop_arg, loop->exit_edges[0]);
add_phi_arg (new_phi, guard_arg, guard_edge);
gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi, e) == loop_arg
|| PHI_ARG_DEF_FROM_EDGE (update_phi, e) == guard_arg);
SET_PHI_ARG_DEF (update_phi, e->dest_idx, PHI_RESULT (new_phi));
}
set_phi_nodes (new_merge_bb, phi_reverse (phi_nodes (new_merge_bb)));
}
void
slpeel_make_loop_iterate_ntimes (struct loop *loop, tree niters)
{
tree indx_before_incr, indx_after_incr, cond_stmt, cond;
tree orig_cond;
edge exit_edge = loop->exit_edges[0];
block_stmt_iterator loop_cond_bsi;
block_stmt_iterator incr_bsi;
bool insert_after;
tree begin_label = tree_block_label (loop->latch);
tree exit_label = tree_block_label (loop->single_exit->dest);
tree init = build_int_cst (TREE_TYPE (niters), 0);
tree step = build_int_cst (TREE_TYPE (niters), 1);
tree then_label;
tree else_label;
LOC loop_loc;
orig_cond = get_loop_exit_condition (loop);
#ifdef ENABLE_CHECKING
gcc_assert (orig_cond);
#endif
loop_cond_bsi = bsi_for_stmt (orig_cond);
standard_iv_increment_position (loop, &incr_bsi, &insert_after);
create_iv (init, step, NULL_TREE, loop,
&incr_bsi, insert_after, &indx_before_incr, &indx_after_incr);
if (exit_edge->flags & EDGE_TRUE_VALUE)
{
cond = build2 (GE_EXPR, boolean_type_node, indx_after_incr, niters);
then_label = build1 (GOTO_EXPR, void_type_node, exit_label);
else_label = build1 (GOTO_EXPR, void_type_node, begin_label);
}
else
{
cond = build2 (LT_EXPR, boolean_type_node, indx_after_incr, niters);
then_label = build1 (GOTO_EXPR, void_type_node, begin_label);
else_label = build1 (GOTO_EXPR, void_type_node, exit_label);
}
cond_stmt = build3 (COND_EXPR, TREE_TYPE (orig_cond), cond,
then_label, else_label);
bsi_insert_before (&loop_cond_bsi, cond_stmt, BSI_SAME_STMT);
bsi_remove (&loop_cond_bsi);
loop_loc = find_loop_location (loop);
if (dump_file && (dump_flags & TDF_DETAILS))
{
if (loop_loc != UNKNOWN_LOC)
fprintf (dump_file, "\nloop at %s:%d: ",
LOC_FILE (loop_loc), LOC_LINE (loop_loc));
print_generic_expr (dump_file, cond_stmt, TDF_SLIM);
}
loop->nb_iterations = niters;
}
static struct loop *
slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *loop, struct loops *loops,
edge e)
{
struct loop *new_loop;
basic_block *new_bbs, *bbs;
bool at_exit;
bool was_imm_dom;
basic_block exit_dest;
tree phi, phi_arg;
at_exit = (e == loop->exit_edges[0]);
if (!at_exit && e != loop_preheader_edge (loop))
return NULL;
bbs = get_loop_body (loop);
if (!can_copy_bbs_p (bbs, loop->num_nodes))
{
free (bbs);
return NULL;
}
new_loop = duplicate_loop (loops, loop, loop->outer);
if (!new_loop)
{
free (bbs);
return NULL;
}
exit_dest = loop->exit_edges[0]->dest;
was_imm_dom = (get_immediate_dominator (CDI_DOMINATORS,
exit_dest) == loop->header ?
true : false);
new_bbs = xmalloc (sizeof (basic_block) * loop->num_nodes);
copy_bbs (bbs, loop->num_nodes, new_bbs, NULL, 0, NULL, NULL);
for (phi = phi_nodes (exit_dest); phi; phi = PHI_CHAIN (phi))
{
phi_arg = PHI_ARG_DEF_FROM_EDGE (phi, loop->exit_edges[0]);
if (phi_arg)
{
edge new_loop_exit_edge;
if (EDGE_SUCC (new_loop->header, 0)->dest == new_loop->latch)
new_loop_exit_edge = EDGE_SUCC (new_loop->header, 1);
else
new_loop_exit_edge = EDGE_SUCC (new_loop->header, 0);
add_phi_arg (phi, phi_arg, new_loop_exit_edge);
}
}
if (at_exit)
{
redirect_edge_and_branch_force (e, new_loop->header);
set_immediate_dominator (CDI_DOMINATORS, new_loop->header, e->src);
if (was_imm_dom)
set_immediate_dominator (CDI_DOMINATORS, exit_dest, new_loop->header);
}
else
{
edge new_exit_e;
edge entry_e = loop_preheader_edge (loop);
basic_block preheader = entry_e->src;
if (!flow_bb_inside_loop_p (new_loop,
EDGE_SUCC (new_loop->header, 0)->dest))
new_exit_e = EDGE_SUCC (new_loop->header, 0);
else
new_exit_e = EDGE_SUCC (new_loop->header, 1);
redirect_edge_and_branch_force (new_exit_e, loop->header);
set_immediate_dominator (CDI_DOMINATORS, loop->header,
new_exit_e->src);
for (phi = phi_nodes (loop->header); phi; phi = PHI_CHAIN (phi))
{
phi_arg = PHI_ARG_DEF_FROM_EDGE (phi, entry_e);
if (phi_arg)
add_phi_arg (phi, phi_arg, new_exit_e);
}
redirect_edge_and_branch_force (entry_e, new_loop->header);
set_immediate_dominator (CDI_DOMINATORS, new_loop->header, preheader);
}
flow_loop_scan (new_loop, LOOP_ALL);
flow_loop_scan (loop, LOOP_ALL);
free (new_bbs);
free (bbs);
return new_loop;
}
static edge
slpeel_add_loop_guard (basic_block guard_bb, tree cond, basic_block exit_bb,
basic_block dom_bb)
{
block_stmt_iterator bsi;
edge new_e, enter_e;
tree cond_stmt, then_label, else_label;
enter_e = EDGE_SUCC (guard_bb, 0);
enter_e->flags &= ~EDGE_FALLTHRU;
enter_e->flags |= EDGE_FALSE_VALUE;
bsi = bsi_last (guard_bb);
then_label = build1 (GOTO_EXPR, void_type_node,
tree_block_label (exit_bb));
else_label = build1 (GOTO_EXPR, void_type_node,
tree_block_label (enter_e->dest));
cond_stmt = build3 (COND_EXPR, void_type_node, cond,
then_label, else_label);
bsi_insert_after (&bsi, cond_stmt, BSI_NEW_STMT);
new_e = make_edge (guard_bb, exit_bb, EDGE_TRUE_VALUE);
set_immediate_dominator (CDI_DOMINATORS, exit_bb, dom_bb);
return new_e;
}
bool
slpeel_can_duplicate_loop_p (struct loop *loop, edge e)
{
edge exit_e = loop->exit_edges [0];
edge entry_e = loop_preheader_edge (loop);
tree orig_cond = get_loop_exit_condition (loop);
block_stmt_iterator loop_exit_bsi = bsi_last (exit_e->src);
if (any_marked_for_rewrite_p ())
return false;
if (loop->inner
|| !loop->outer
|| loop->num_nodes != 2
|| !empty_block_p (loop->latch)
|| loop->num_exits != 1
|| loop->num_entries != 1
|| (!orig_cond || orig_cond != bsi_stmt (loop_exit_bsi))
|| (e != exit_e && e != entry_e))
return false;
return true;
}
#ifdef ENABLE_CHECKING
void
slpeel_verify_cfg_after_peeling (struct loop *first_loop,
struct loop *second_loop)
{
basic_block loop1_exit_bb = first_loop->exit_edges[0]->dest;
basic_block loop2_entry_bb = second_loop->pre_header;
basic_block loop1_entry_bb = loop_preheader_edge (first_loop)->src;
gcc_assert (EDGE_COUNT (loop1_exit_bb->succs) == 2);
gcc_assert (EDGE_COUNT (loop2_entry_bb->preds) == 2
&& ((EDGE_PRED (loop2_entry_bb, 0)->src == loop1_exit_bb
&& EDGE_PRED (loop2_entry_bb, 1)->src == loop1_entry_bb)
|| (EDGE_PRED (loop2_entry_bb, 1)->src == loop1_exit_bb
&& EDGE_PRED (loop2_entry_bb, 0)->src == loop1_entry_bb)));
}
#endif
struct loop*
slpeel_tree_peel_loop_to_edge (struct loop *loop, struct loops *loops,
edge e, tree first_niters,
tree niters, bool update_first_loop_count)
{
struct loop *new_loop = NULL, *first_loop, *second_loop;
edge skip_e;
tree pre_condition;
bitmap definitions;
basic_block bb_before_second_loop, bb_after_second_loop;
basic_block bb_before_first_loop;
basic_block bb_between_loops;
edge exit_e = loop->exit_edges [0];
LOC loop_loc;
if (!slpeel_can_duplicate_loop_p (loop, e))
return NULL;
tree_register_cfg_hooks ();
if (!(new_loop = slpeel_tree_duplicate_loop_to_edge_cfg (loop, loops, e)))
{
loop_loc = find_loop_location (loop);
if (dump_file && (dump_flags & TDF_DETAILS))
{
if (loop_loc != UNKNOWN_LOC)
fprintf (dump_file, "\n%s:%d: note: ",
LOC_FILE (loop_loc), LOC_LINE (loop_loc));
fprintf (dump_file, "tree_duplicate_loop_to_edge_cfg failed.\n");
}
return NULL;
}
if (e == exit_e)
{
first_loop = loop;
second_loop = new_loop;
}
else
{
first_loop = new_loop;
second_loop = loop;
}
definitions = marked_ssa_names ();
allocate_new_names (definitions);
slpeel_update_phis_for_duplicate_loop (loop, new_loop, e == exit_e);
rename_variables_in_loop (new_loop);
bb_before_first_loop = split_edge (loop_preheader_edge (first_loop));
add_bb_to_loop (bb_before_first_loop, first_loop->outer);
bb_before_second_loop = split_edge (first_loop->exit_edges[0]);
add_bb_to_loop (bb_before_second_loop, first_loop->outer);
flow_loop_scan (first_loop, LOOP_ALL);
flow_loop_scan (second_loop, LOOP_ALL);
pre_condition =
build2 (LE_EXPR, boolean_type_node, first_niters, integer_zero_node);
skip_e = slpeel_add_loop_guard (bb_before_first_loop, pre_condition,
bb_before_second_loop, bb_before_first_loop);
slpeel_update_phi_nodes_for_guard (skip_e, first_loop, true ,
first_loop == new_loop);
bb_between_loops = split_edge (first_loop->exit_edges[0]);
add_bb_to_loop (bb_between_loops, first_loop->outer);
bb_after_second_loop = split_edge (second_loop->exit_edges[0]);
add_bb_to_loop (bb_after_second_loop, second_loop->outer);
flow_loop_scan (first_loop, LOOP_ALL);
flow_loop_scan (second_loop, LOOP_ALL);
pre_condition = build2 (EQ_EXPR, boolean_type_node, first_niters, niters);
skip_e = slpeel_add_loop_guard (bb_between_loops, pre_condition,
bb_after_second_loop, bb_before_first_loop);
slpeel_update_phi_nodes_for_guard (skip_e, second_loop, false ,
second_loop == new_loop);
first_loop->single_exit = first_loop->exit_edges[0];
second_loop->single_exit = second_loop->exit_edges[0];
if (update_first_loop_count)
slpeel_make_loop_iterate_ntimes (first_loop, first_niters);
free_new_names (definitions);
BITMAP_FREE (definitions);
unmark_all_for_rewrite ();
return new_loop;
}
LOC
find_loop_location (struct loop *loop)
{
tree node = NULL_TREE;
basic_block bb;
block_stmt_iterator si;
if (!loop)
return UNKNOWN_LOC;
node = get_loop_exit_condition (loop);
if (node && EXPR_P (node) && EXPR_HAS_LOCATION (node)
&& EXPR_FILENAME (node) && EXPR_LINENO (node))
return EXPR_LOC (node);
if (!loop->header)
return UNKNOWN_LOC;
bb = loop->header;
for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
{
node = bsi_stmt (si);
if (node && EXPR_P (node) && EXPR_HAS_LOCATION (node))
return EXPR_LOC (node);
}
return UNKNOWN_LOC;
}
void
vect_set_verbosity_level (const char *val)
{
unsigned int vl;
vl = atoi (val);
if (vl < MAX_VERBOSITY_LEVEL)
vect_verbosity_level = vl;
else
vect_verbosity_level = MAX_VERBOSITY_LEVEL - 1;
}
static void
vect_set_dump_settings (void)
{
vect_dump = dump_file;
if (vect_verbosity_level != MAX_VERBOSITY_LEVEL)
{
if (!dump_file)
vect_dump = stderr;
return;
}
if (dump_file && (dump_flags & TDF_DETAILS))
vect_verbosity_level = REPORT_DETAILS;
else if (dump_file && (dump_flags & TDF_STATS))
vect_verbosity_level = REPORT_UNVECTORIZED_LOOPS;
else
vect_verbosity_level = REPORT_NONE;
gcc_assert (dump_file || vect_verbosity_level == REPORT_NONE);
}
bool
vect_print_dump_info (enum verbosity_levels vl, LOC loc ATTRIBUTE_UNUSED)
{
if (vl > vect_verbosity_level)
return false;
if (!vect_dump)
return false;
fprintf (vect_dump, "\n%s:%d: note: ", LOC_FILE (vect_loop_location),
LOC_LINE (vect_loop_location));
return true;
}
stmt_vec_info
new_stmt_vec_info (tree stmt, loop_vec_info loop_vinfo)
{
stmt_vec_info res;
res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
STMT_VINFO_TYPE (res) = undef_vec_info_type;
STMT_VINFO_STMT (res) = stmt;
STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
STMT_VINFO_RELEVANT_P (res) = 0;
STMT_VINFO_VECTYPE (res) = NULL;
STMT_VINFO_VEC_STMT (res) = NULL;
STMT_VINFO_DATA_REF (res) = NULL;
STMT_VINFO_MEMTAG (res) = NULL;
STMT_VINFO_VECT_DR_BASE_ADDRESS (res) = NULL;
STMT_VINFO_VECT_INIT_OFFSET (res) = NULL_TREE;
STMT_VINFO_VECT_STEP (res) = NULL_TREE;
STMT_VINFO_VECT_BASE_ALIGNED_P (res) = false;
STMT_VINFO_VECT_MISALIGNMENT (res) = NULL_TREE;
return res;
}
loop_vec_info
new_loop_vec_info (struct loop *loop)
{
loop_vec_info res;
basic_block *bbs;
block_stmt_iterator si;
unsigned int i;
res = (loop_vec_info) xcalloc (1, sizeof (struct _loop_vec_info));
bbs = get_loop_body (loop);
for (i = 0; i < loop->num_nodes; i++)
{
basic_block bb = bbs[i];
for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
{
tree stmt = bsi_stmt (si);
stmt_ann_t ann;
get_stmt_operands (stmt);
ann = stmt_ann (stmt);
set_stmt_info (ann, new_stmt_vec_info (stmt, res));
}
}
LOOP_VINFO_LOOP (res) = loop;
LOOP_VINFO_BBS (res) = bbs;
LOOP_VINFO_EXIT_COND (res) = NULL;
LOOP_VINFO_NITERS (res) = NULL;
LOOP_VINFO_VECTORIZABLE_P (res) = 0;
LOOP_DO_PEELING_FOR_ALIGNMENT (res) = false;
LOOP_VINFO_VECT_FACTOR (res) = 0;
VARRAY_GENERIC_PTR_INIT (LOOP_VINFO_DATAREF_WRITES (res), 20,
"loop_write_datarefs");
VARRAY_GENERIC_PTR_INIT (LOOP_VINFO_DATAREF_READS (res), 20,
"loop_read_datarefs");
LOOP_VINFO_UNALIGNED_DR (res) = NULL;
LOOP_VINFO_LOC (res) = UNKNOWN_LOC;
return res;
}
void
destroy_loop_vec_info (loop_vec_info loop_vinfo)
{
struct loop *loop;
basic_block *bbs;
int nbbs;
block_stmt_iterator si;
int j;
if (!loop_vinfo)
return;
loop = LOOP_VINFO_LOOP (loop_vinfo);
bbs = LOOP_VINFO_BBS (loop_vinfo);
nbbs = loop->num_nodes;
for (j = 0; j < nbbs; j++)
{
basic_block bb = bbs[j];
for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
{
tree stmt = bsi_stmt (si);
stmt_ann_t ann = stmt_ann (stmt);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
free (stmt_info);
set_stmt_info (ann, NULL);
}
}
free (LOOP_VINFO_BBS (loop_vinfo));
varray_clear (LOOP_VINFO_DATAREF_WRITES (loop_vinfo));
varray_clear (LOOP_VINFO_DATAREF_READS (loop_vinfo));
free (loop_vinfo);
}
tree
vect_strip_conversion (tree expr)
{
tree to, ti, oprnd0;
while (TREE_CODE (expr) == NOP_EXPR || TREE_CODE (expr) == CONVERT_EXPR)
{
to = TREE_TYPE (expr);
oprnd0 = TREE_OPERAND (expr, 0);
ti = TREE_TYPE (oprnd0);
if (!INTEGRAL_TYPE_P (to) || !INTEGRAL_TYPE_P (ti))
return NULL_TREE;
if (GET_MODE_SIZE (TYPE_MODE (to)) < GET_MODE_SIZE (TYPE_MODE (ti)))
return NULL_TREE;
expr = oprnd0;
}
return expr;
}
bool
vect_can_force_dr_alignment_p (tree decl, unsigned int alignment)
{
if (TREE_CODE (decl) != VAR_DECL)
return false;
if (DECL_EXTERNAL (decl))
return false;
if (TREE_ASM_WRITTEN (decl))
return false;
if (TREE_STATIC (decl))
return (alignment <= MAX_OFILE_ALIGNMENT);
else
return (alignment <= PREFERRED_STACK_BOUNDARY);
}
tree
get_vectype_for_scalar_type (tree scalar_type)
{
enum machine_mode inner_mode = TYPE_MODE (scalar_type);
int nbytes = GET_MODE_SIZE (inner_mode);
int nunits;
tree vectype;
if (nbytes == 0)
return NULL_TREE;
nunits = UNITS_PER_SIMD_WORD / nbytes;
vectype = build_vector_type (scalar_type, nunits);
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
{
fprintf (vect_dump, "get vectype with %d units of type ", nunits);
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
}
if (!vectype)
return NULL_TREE;
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
{
fprintf (vect_dump, "vectype: ");
print_generic_expr (vect_dump, vectype, TDF_SLIM);
}
if (!VECTOR_MODE_P (TYPE_MODE (vectype)))
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "mode not supported by target.");
return NULL_TREE;
}
return vectype;
}
enum dr_alignment_support
vect_supportable_dr_alignment (struct data_reference *dr)
{
tree vectype = STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr)));
enum machine_mode mode = (int) TYPE_MODE (vectype);
if (aligned_access_p (dr))
return dr_aligned;
if (DR_IS_READ (dr))
{
if (vec_realign_load_optab->handlers[mode].insn_code != CODE_FOR_nothing
&& (!targetm.vectorize.builtin_mask_for_load
|| targetm.vectorize.builtin_mask_for_load ()))
return dr_unaligned_software_pipeline;
if (movmisalign_optab->handlers[mode].insn_code != CODE_FOR_nothing)
return dr_unaligned_supported;
}
return dr_unaligned_unsupported;
}
bool
vect_is_simple_use (tree operand, loop_vec_info loop_vinfo, tree *def)
{
tree def_stmt;
basic_block bb;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
if (def)
*def = NULL_TREE;
if (TREE_CODE (operand) == INTEGER_CST || TREE_CODE (operand) == REAL_CST)
return true;
if (TREE_CODE (operand) != SSA_NAME)
return false;
def_stmt = SSA_NAME_DEF_STMT (operand);
if (def_stmt == NULL_TREE )
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "no def_stmt.");
return false;
}
if (IS_EMPTY_STMT (def_stmt))
{
tree arg = TREE_OPERAND (def_stmt, 0);
if (TREE_CODE (arg) == INTEGER_CST || TREE_CODE (arg) == REAL_CST)
return true;
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
{
fprintf (vect_dump, "Unexpected empty stmt: ");
print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
}
return false;
}
bb = bb_for_stmt (def_stmt);
if (TREE_CODE (def_stmt) == PHI_NODE && flow_bb_inside_loop_p (loop, bb))
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "reduction/induction - unsupported.");
return false;
}
if (TREE_CODE (def_stmt) == MODIFY_EXPR
|| TREE_CODE (def_stmt) == PHI_NODE)
{
if (def)
*def = def_stmt;
return true;
}
return false;
}
bool
vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
tree * step)
{
tree init_expr;
tree step_expr;
tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
if (evolution_part == NULL_TREE)
return false;
if (tree_is_chrec (evolution_part))
return false;
step_expr = evolution_part;
init_expr = unshare_expr (initial_condition_in_loop_num (access_fn,
loop_nb));
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
{
fprintf (vect_dump, "step: ");
print_generic_expr (vect_dump, step_expr, TDF_SLIM);
fprintf (vect_dump, ", init: ");
print_generic_expr (vect_dump, init_expr, TDF_SLIM);
}
*init = init_expr;
*step = step_expr;
if (TREE_CODE (step_expr) != INTEGER_CST)
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "step unknown.");
return false;
}
return true;
}
static bool
need_imm_uses_for (tree var)
{
return is_gimple_reg (var);
}
void
vectorize_loops (struct loops *loops)
{
unsigned int i;
unsigned int num_vectorized_loops = 0;
vect_set_dump_settings ();
if (!UNITS_PER_SIMD_WORD)
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "vectorizer: target vector size is not defined.");
return;
}
#ifdef ENABLE_CHECKING
verify_loop_closed_ssa ();
#endif
compute_immediate_uses (TDFA_USE_OPS, need_imm_uses_for);
loops_num = loops->num;
for (i = 1; i < loops_num; i++)
{
loop_vec_info loop_vinfo;
struct loop *loop = loops->parray[i];
if (!loop)
continue;
vect_loop_location = find_loop_location (loop);
loop_vinfo = vect_analyze_loop (loop);
loop->aux = loop_vinfo;
if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
continue;
vect_transform_loop (loop_vinfo, loops);
DECL_STRUCT_FUNCTION (current_function_decl)->uses_vector = 1;
num_vectorized_loops++;
}
if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS, UNKNOWN_LOC))
fprintf (vect_dump, "vectorized %u loops in function.\n",
num_vectorized_loops);
free_df ();
for (i = 1; i < loops_num; i++)
{
struct loop *loop = loops->parray[i];
loop_vec_info loop_vinfo;
if (!loop)
continue;
loop_vinfo = loop->aux;
destroy_loop_vec_info (loop_vinfo);
loop->aux = NULL;
}
rewrite_into_ssa (false);
rewrite_into_loop_closed_ssa ();
bitmap_clear (vars_to_rename);
}