#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "rtl.h"
#include "expr.h"
#include "hard-reg-set.h"
#include "basic-block.h"
#include "value-prof.h"
#include "output.h"
#include "flags.h"
#include "insn-config.h"
#include "recog.h"
#include "optabs.h"
#include "regs.h"
#include "ggc.h"
static struct value_prof_hooks *value_prof_hooks;
#ifndef NOPREFETCH_RANGE_MIN
#define NOPREFETCH_RANGE_MIN (-16)
#endif
#ifndef NOPREFETCH_RANGE_MAX
#define NOPREFETCH_RANGE_MAX 32
#endif
static void insn_divmod_values_to_profile (rtx, histogram_values *);
#ifdef HAVE_prefetch
static bool insn_prefetch_values_to_profile (rtx, histogram_values *);
static int find_mem_reference_1 (rtx *, void *);
static void find_mem_reference_2 (rtx, rtx, void *);
static bool find_mem_reference (rtx, rtx *, int *);
#endif
static void insn_values_to_profile (rtx, histogram_values *);
static rtx gen_divmod_fixed_value (enum machine_mode, enum rtx_code, rtx, rtx,
rtx, gcov_type, int);
static rtx gen_mod_pow2 (enum machine_mode, enum rtx_code, rtx, rtx, rtx, int);
static rtx gen_mod_subtract (enum machine_mode, enum rtx_code, rtx, rtx, rtx,
int, int, int);
#ifdef HAVE_prefetch
static rtx gen_speculative_prefetch (rtx, gcov_type, int);
#endif
static bool divmod_fixed_value_transform (rtx insn);
static bool mod_pow2_value_transform (rtx);
static bool mod_subtract_transform (rtx);
#ifdef HAVE_prefetch
static bool speculative_prefetching_transform (rtx);
#endif
static void
insn_divmod_values_to_profile (rtx insn, histogram_values *values)
{
rtx set, set_src, op1, op2;
enum machine_mode mode;
histogram_value hist;
if (!INSN_P (insn))
return;
set = single_set (insn);
if (!set)
return;
mode = GET_MODE (SET_DEST (set));
if (!INTEGRAL_MODE_P (mode))
return;
set_src = SET_SRC (set);
switch (GET_CODE (set_src))
{
case DIV:
case MOD:
case UDIV:
case UMOD:
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
if (side_effects_p (op2))
return;
if ((GET_CODE (set_src) == UMOD) && !CONSTANT_P (op2))
{
hist = ggc_alloc (sizeof (*hist));
hist->value = op2;
hist->seq = NULL_RTX;
hist->mode = mode;
hist->insn = insn;
hist->type = HIST_TYPE_POW2;
hist->hdata.pow2.may_be_other = 1;
VEC_safe_push (histogram_value, *values, hist);
}
if (!CONSTANT_P (op2))
{
hist = ggc_alloc (sizeof (*hist));
hist->value = op2;
hist->mode = mode;
hist->seq = NULL_RTX;
hist->insn = insn;
hist->type = HIST_TYPE_SINGLE_VALUE;
VEC_safe_push (histogram_value, *values, hist);
}
if (GET_CODE (set_src) == UMOD && !side_effects_p (op1))
{
rtx tmp;
hist = ggc_alloc (sizeof (*hist));
start_sequence ();
tmp = simplify_gen_binary (DIV, mode, copy_rtx (op1), copy_rtx (op2));
hist->value = force_operand (tmp, NULL_RTX);
hist->seq = get_insns ();
end_sequence ();
hist->mode = mode;
hist->insn = insn;
hist->type = HIST_TYPE_INTERVAL;
hist->hdata.intvl.int_start = 0;
hist->hdata.intvl.steps = 2;
hist->hdata.intvl.may_be_less = 1;
hist->hdata.intvl.may_be_more = 1;
VEC_safe_push (histogram_value, *values, hist);
}
return;
default:
return;
}
}
#ifdef HAVE_prefetch
static int
find_mem_reference_1 (rtx *expr, void *ret)
{
rtx *mem = ret;
if (GET_CODE (*expr) == MEM)
{
*mem = *expr;
return 1;
}
return 0;
}
static int fmr2_write;
static void
find_mem_reference_2 (rtx expr, rtx pat ATTRIBUTE_UNUSED, void *mem)
{
if (expr == mem)
fmr2_write = true;
}
static bool
find_mem_reference (rtx insn, rtx *mem, int *write)
{
*mem = NULL_RTX;
for_each_rtx (&PATTERN (insn), find_mem_reference_1, mem);
if (!*mem)
return false;
fmr2_write = false;
note_stores (PATTERN (insn), find_mem_reference_2, *mem);
*write = fmr2_write;
return true;
}
static bool
insn_prefetch_values_to_profile (rtx insn, histogram_values *values)
{
rtx mem, address;
int write;
histogram_value hist;
if (GET_CODE (insn) != INSN)
return false;
if (!find_mem_reference (insn, &mem, &write))
return false;
address = XEXP (mem, 0);
if (side_effects_p (address))
return false;
if (GET_CODE (PATTERN (insn)) == CLOBBER)
return false;
if (CONSTANT_P (address))
return false;
hist = ggc_alloc (sizeof (*hist));
hist->value = address;
hist->mode = GET_MODE (address);
hist->seq = NULL_RTX;
hist->insn = insn;
hist->type = HIST_TYPE_CONST_DELTA;
VEC_safe_push (histogram_value, *values, hist);
return true;
}
#endif
static void
insn_values_to_profile (rtx insn, histogram_values *values)
{
if (flag_value_profile_transformations)
insn_divmod_values_to_profile (insn, values);
#ifdef HAVE_prefetch
if (flag_speculative_prefetching)
insn_prefetch_values_to_profile (insn, values);
#endif
}
static void
rtl_find_values_to_profile (histogram_values *values)
{
rtx insn;
unsigned i, libcall_level;
life_analysis (NULL, PROP_DEATH_NOTES);
*values = VEC_alloc (histogram_value, 0);
libcall_level = 0;
for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
{
if (find_reg_note (insn, REG_LIBCALL, NULL_RTX))
libcall_level++;
if (!libcall_level)
insn_values_to_profile (insn, values);
if (find_reg_note (insn, REG_RETVAL, NULL_RTX))
{
gcc_assert (libcall_level > 0);
libcall_level--;
}
}
gcc_assert (libcall_level == 0);
for (i = 0; i < VEC_length (histogram_value, *values); i++)
{
histogram_value hist = VEC_index (histogram_value, *values, i);
switch (hist->type)
{
case HIST_TYPE_INTERVAL:
if (dump_file)
fprintf (dump_file,
"Interval counter for insn %d, range %d -- %d.\n",
INSN_UID ((rtx)hist->insn),
hist->hdata.intvl.int_start,
(hist->hdata.intvl.int_start
+ hist->hdata.intvl.steps - 1));
hist->n_counters = hist->hdata.intvl.steps +
(hist->hdata.intvl.may_be_less ? 1 : 0) +
(hist->hdata.intvl.may_be_more ? 1 : 0);
break;
case HIST_TYPE_POW2:
if (dump_file)
fprintf (dump_file,
"Pow2 counter for insn %d.\n",
INSN_UID ((rtx)hist->insn));
hist->n_counters
= GET_MODE_BITSIZE (hist->mode)
+ (hist->hdata.pow2.may_be_other ? 1 : 0);
break;
case HIST_TYPE_SINGLE_VALUE:
if (dump_file)
fprintf (dump_file,
"Single value counter for insn %d.\n",
INSN_UID ((rtx)hist->insn));
hist->n_counters = 3;
break;
case HIST_TYPE_CONST_DELTA:
if (dump_file)
fprintf (dump_file,
"Constant delta counter for insn %d.\n",
INSN_UID ((rtx)hist->insn));
hist->n_counters = 4;
break;
default:
abort ();
}
}
allocate_reg_info (max_reg_num (), FALSE, FALSE);
}
static bool
rtl_value_profile_transformations (void)
{
rtx insn, next;
int changed = false;
for (insn = get_insns (); insn; insn = next)
{
next = NEXT_INSN (insn);
if (!INSN_P (insn))
continue;
if (!find_reg_note (insn, REG_VALUE_PROFILE, 0))
continue;
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
continue;
if (dump_file)
{
fprintf (dump_file, "Trying transformations on insn %d\n",
INSN_UID (insn));
print_rtl_single (dump_file, insn);
}
if (flag_value_profile_transformations
&& (mod_subtract_transform (insn)
|| divmod_fixed_value_transform (insn)
|| mod_pow2_value_transform (insn)))
changed = true;
#ifdef HAVE_prefetch
if (flag_speculative_prefetching
&& speculative_prefetching_transform (insn))
changed = true;
#endif
}
if (changed)
{
commit_edge_insertions ();
allocate_reg_info (max_reg_num (), FALSE, FALSE);
}
return changed;
}
static rtx
gen_divmod_fixed_value (enum machine_mode mode, enum rtx_code operation,
rtx target, rtx op1, rtx op2, gcov_type value,
int prob)
{
rtx tmp, tmp1, jump;
rtx neq_label = gen_label_rtx ();
rtx end_label = gen_label_rtx ();
rtx sequence;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
do_compare_rtx_and_jump (tmp, GEN_INT (value), NE, 0, mode, NULL_RTX,
NULL_RTX, neq_label);
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (REG_BR_PROB_BASE - prob),
REG_NOTES (jump));
tmp1 = simplify_gen_binary (operation, mode,
copy_rtx (op1), GEN_INT (value));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (copy_rtx (target), copy_rtx (tmp1));
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
emit_label (neq_label);
tmp1 = simplify_gen_binary (operation, mode,
copy_rtx (op1), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (copy_rtx (target), copy_rtx (tmp1));
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static bool
divmod_fixed_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type val, count, all;
edge e;
int prob;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != DIV && code != MOD && code != UDIV && code != UMOD)
return false;
op1 = XEXP (set_src, false);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_SINGLE_VALUE))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
if (!rtx_equal_p (op2, value) || 2 * count < all)
return false;
if (dump_file)
fprintf (dump_file, "Div/mod by constant transformation on insn %d\n",
INSN_UID (insn));
prob = (count * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_divmod_fixed_value (mode, code, set_dest,
op1, op2, val, prob), e);
return true;
}
static rtx
gen_mod_pow2 (enum machine_mode mode, enum rtx_code operation, rtx target,
rtx op1, rtx op2, int prob)
{
rtx tmp, tmp1, tmp2, tmp3, jump;
rtx neq_label = gen_label_rtx ();
rtx end_label = gen_label_rtx ();
rtx sequence;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
tmp1 = expand_simple_binop (mode, PLUS, tmp, constm1_rtx, NULL_RTX,
0, OPTAB_WIDEN);
tmp2 = expand_simple_binop (mode, AND, tmp, tmp1, NULL_RTX,
0, OPTAB_WIDEN);
do_compare_rtx_and_jump (tmp2, const0_rtx, NE, 0, mode, NULL_RTX,
NULL_RTX, neq_label);
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (REG_BR_PROB_BASE - prob),
REG_NOTES (jump));
tmp3 = expand_simple_binop (mode, AND, op1, tmp1, target,
0, OPTAB_WIDEN);
if (tmp3 != target)
emit_move_insn (copy_rtx (target), tmp3);
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
emit_label (neq_label);
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (op1), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (target, tmp1);
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static bool
mod_pow2_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, count;
edge e;
int i, all, prob;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_POW2))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
wrong_values =INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = 0;
for (i = 0; i < GET_MODE_BITSIZE (mode); i++)
{
count += INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
}
if (!rtx_equal_p (op2, value))
return false;
if (count < wrong_values)
return false;
if (dump_file)
fprintf (dump_file, "Mod power of 2 transformation on insn %d\n",
INSN_UID (insn));
all = count + wrong_values;
prob = (count * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_pow2 (mode, code, set_dest, op1, op2, prob), e);
return true;
}
static rtx
gen_mod_subtract (enum machine_mode mode, enum rtx_code operation,
rtx target, rtx op1, rtx op2, int sub, int prob1, int prob2)
{
rtx tmp, tmp1, jump;
rtx end_label = gen_label_rtx ();
rtx sequence;
int i;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
emit_move_insn (target, copy_rtx (op1));
do_compare_rtx_and_jump (target, tmp, LTU, 0, mode, NULL_RTX,
NULL_RTX, end_label);
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (prob1), REG_NOTES (jump));
for (i = 0; i < sub; i++)
{
tmp1 = expand_simple_binop (mode, MINUS, target, tmp, target,
0, OPTAB_WIDEN);
if (tmp1 != target)
emit_move_insn (target, tmp1);
do_compare_rtx_and_jump (target, tmp, LTU, 0, mode, NULL_RTX,
NULL_RTX, end_label);
jump = get_last_insn ();
REG_NOTES (jump) = gen_rtx_EXPR_LIST (REG_BR_PROB,
GEN_INT (prob2), REG_NOTES (jump));
}
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (target), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (target, tmp1);
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static bool
mod_subtract_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, counts[2], count, all;
edge e;
int i, prob1, prob2;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_INTERVAL))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
all = 0;
for (i = 0; i < 2; i++)
{
counts[i] = INTVAL (XEXP (histogram, 0));
all += counts[i];
histogram = XEXP (histogram, 1);
}
wrong_values = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
wrong_values += INTVAL (XEXP (histogram, 0));
all += wrong_values;
count = 0;
for (i = 0; i < 2; i++)
{
count += counts[i];
if (count * 2 >= all)
break;
}
if (i == 2)
return false;
if (dump_file)
fprintf (dump_file, "Mod subtract transformation on insn %d\n",
INSN_UID (insn));
prob1 = (counts[0] * REG_BR_PROB_BASE + all / 2) / all;
prob2 = (counts[1] * REG_BR_PROB_BASE + all / 2) / all;
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_subtract (mode, code, set_dest,
op1, op2, i, prob1, prob2), e);
return true;
}
#ifdef HAVE_prefetch
static rtx
gen_speculative_prefetch (rtx address, gcov_type delta, int write)
{
rtx tmp;
rtx sequence;
start_sequence ();
if (offsettable_address_p (0, VOIDmode, address))
tmp = plus_constant (copy_rtx (address), delta);
else
{
tmp = simplify_gen_binary (PLUS, Pmode,
copy_rtx (address), GEN_INT (delta));
tmp = force_operand (tmp, NULL);
}
if (! (*insn_data[(int)CODE_FOR_prefetch].operand[0].predicate)
(tmp, insn_data[(int)CODE_FOR_prefetch].operand[0].mode))
tmp = force_reg (Pmode, tmp);
emit_insn (gen_prefetch (tmp, GEN_INT (write), GEN_INT (3)));
sequence = get_insns ();
end_sequence ();
return sequence;
}
static bool
speculative_prefetching_transform (rtx insn)
{
rtx histogram, value;
gcov_type val, count, all;
edge e;
rtx mem, address;
int write;
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
return false;
if (!find_mem_reference (insn, &mem, &write))
return false;
address = XEXP (mem, 0);
if (side_effects_p (address))
return false;
if (CONSTANT_P (address))
return false;
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_CONST_DELTA))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
if (all < 4)
return false;
if (!rtx_equal_p (address, value) || 2 * count < all)
return false;
if (val >= NOPREFETCH_RANGE_MIN && val <= NOPREFETCH_RANGE_MAX)
return false;
if (dump_file)
fprintf (dump_file, "Speculative prefetching for insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
insert_insn_on_edge (gen_speculative_prefetch (address, val, write), e);
return true;
}
#endif
struct value_prof_hooks {
void (*find_values_to_profile) (histogram_values *);
bool (*value_profile_transformations) (void);
};
static struct value_prof_hooks rtl_value_prof_hooks =
{
rtl_find_values_to_profile,
rtl_value_profile_transformations
};
void
rtl_register_value_prof_hooks (void)
{
value_prof_hooks = &rtl_value_prof_hooks;
if (ir_type ())
abort ();
}
static void
tree_find_values_to_profile (histogram_values *values ATTRIBUTE_UNUSED)
{
abort ();
}
static bool
tree_value_profile_transformations (void)
{
abort ();
}
static struct value_prof_hooks tree_value_prof_hooks = {
tree_find_values_to_profile,
tree_value_profile_transformations
};
void
tree_register_value_prof_hooks (void)
{
value_prof_hooks = &tree_value_prof_hooks;
if (!ir_type ())
abort ();
}
void
find_values_to_profile (histogram_values *values)
{
(value_prof_hooks->find_values_to_profile) (values);
}
bool
value_profile_transformations (void)
{
return (value_prof_hooks->value_profile_transformations) ();
}