From: Aaron Sawdey Date: Tue, 5 Feb 2019 16:32:06 +0000 (+0000) Subject: re PR target/89112 (Incorrect code generated by rs6000 memcmp expansion) X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=faaeebd64671ebe0aabf203527bb7bf57476f4f3;p=gcc.git re PR target/89112 (Incorrect code generated by rs6000 memcmp expansion) 2019-02-05 Aaron Sawdey PR target/89112 * config/rs6000/rs6000-string.c (do_ifelse, expand_cmp_vec_sequence, expand_compare_loop, expand_block_compare_gpr, expand_strncmp_align_check, expand_strncmp_gpr_sequence): Insert REG_BR_PROB notes in inline expansion of memcmp/strncmp. Add #include "profile-count.h" and "predict.h" for types and functions needed to work with REG_BR_PROB notes. From-SVN: r268547 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index cbc8b3b242e..b3c2e7c46d3 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,13 @@ +2019-02-05 Aaron Sawdey + + PR target/89112 + * config/rs6000/rs6000-string.c (do_ifelse, expand_cmp_vec_sequence, + expand_compare_loop, expand_block_compare_gpr, + expand_strncmp_align_check, expand_strncmp_gpr_sequence): Insert + REG_BR_PROB notes in inline expansion of memcmp/strncmp. Add + #include "profile-count.h" and "predict.h" for types and functions + needed to work with REG_BR_PROB notes. + 2019-02-05 Aaron Sawdey PR target/89112 diff --git a/gcc/config/rs6000/rs6000-string.c b/gcc/config/rs6000/rs6000-string.c index 46d6f6b5849..7101dc77e08 100644 --- a/gcc/config/rs6000/rs6000-string.c +++ b/gcc/config/rs6000/rs6000-string.c @@ -35,6 +35,8 @@ #include "expr.h" #include "output.h" #include "target.h" +#include "profile-count.h" +#include "predict.h" /* Expand a block clear operation, and return 1 if successful. Return 0 if we should let the compiler generate normal code. @@ -369,6 +371,7 @@ do_load_for_compare_from_addr (machine_mode mode, rtx dest, rtx addr, B is the second thing to be compared. CR is the condition code reg input, or NULL_RTX. TRUE_LABEL is the label to branch to if the condition is true. + P is the estimated branch probability for the branch. The return value is the CR used for the comparison. If CR is null_rtx, then a new register of CMPMODE is generated. @@ -377,7 +380,7 @@ do_load_for_compare_from_addr (machine_mode mode, rtx dest, rtx addr, static void do_ifelse (machine_mode cmpmode, rtx_code comparison, - rtx a, rtx b, rtx cr, rtx true_label) + rtx a, rtx b, rtx cr, rtx true_label, profile_probability br_prob) { gcc_assert ((a == NULL_RTX && b == NULL_RTX && cr != NULL_RTX) || (a != NULL_RTX && b != NULL_RTX)); @@ -395,7 +398,8 @@ do_ifelse (machine_mode cmpmode, rtx_code comparison, rtx cmp_rtx = gen_rtx_fmt_ee (comparison, VOIDmode, cr, const0_rtx); rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, label_ref, pc_rtx); - rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + add_reg_br_prob_note (j, br_prob); JUMP_LABEL (j) = true_label; LABEL_NUSES (true_label) += 1; } @@ -781,7 +785,8 @@ expand_cmp_vec_sequence (unsigned HOST_WIDE_INT bytes_to_compare, rtx lab_ref = gen_rtx_LABEL_REF (VOIDmode, dst_label); rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, lab_ref, pc_rtx); - rtx j2 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + rtx_insn *j2 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + add_reg_br_prob_note (j2, profile_probability::likely ()); JUMP_LABEL (j2) = dst_label; LABEL_NUSES (dst_label) += 1; @@ -1036,7 +1041,7 @@ expand_compare_loop (rtx operands[]) /* Difference found is stored here before jump to diff_label. */ rtx diff = gen_reg_rtx (word_mode); - rtx j; + rtx_insn *j; /* Example of generated code for 35 bytes aligned 1 byte. @@ -1120,11 +1125,11 @@ expand_compare_loop (rtx operands[]) /* Check for > max_bytes bytes. We want to bail out as quickly as possible if we have to go over to memcmp. */ do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (max_bytes), - NULL_RTX, library_call_label); + NULL_RTX, library_call_label, profile_probability::even ()); /* Check for < loop_bytes bytes. */ do_ifelse (CCmode, LT, bytes_rtx, GEN_INT (loop_bytes), - NULL_RTX, cleanup_label); + NULL_RTX, cleanup_label, profile_probability::even ()); /* Loop compare bytes and iterations if bytes>max_bytes. */ rtx mb_reg = gen_reg_rtx (word_mode); @@ -1165,7 +1170,7 @@ expand_compare_loop (rtx operands[]) { rtx lab_after = gen_label_rtx (); do_ifelse (CCmode, LE, bytes_rtx, GEN_INT (max_bytes), - NULL_RTX, lab_after); + NULL_RTX, lab_after, profile_probability::even ()); emit_move_insn (loop_cmp, mb_reg); emit_move_insn (iter, mi_reg); emit_label (lab_after); @@ -1236,7 +1241,7 @@ expand_compare_loop (rtx operands[]) } do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX, - dcond, diff_label); + dcond, diff_label, profile_probability::unlikely ()); if (TARGET_P9_MISC) { @@ -1260,6 +1265,7 @@ expand_compare_loop (rtx operands[]) else j = emit_jump_insn (gen_bdnztf_si (loop_top_label, ctr, ctr, eqrtx, dcond)); + add_reg_br_prob_note (j, profile_probability::likely ()); JUMP_LABEL (j) = loop_top_label; LABEL_NUSES (loop_top_label) += 1; } @@ -1272,9 +1278,11 @@ expand_compare_loop (rtx operands[]) code. If we exit here with a nonzero diff, it is because the second word differed. */ if (TARGET_P9_MISC) - do_ifelse (CCUNSmode, NE, NULL_RTX, NULL_RTX, dcond, diff_label); + do_ifelse (CCUNSmode, NE, NULL_RTX, NULL_RTX, dcond, + diff_label, profile_probability::unlikely ()); else - do_ifelse (CCmode, NE, diff, const0_rtx, NULL_RTX, diff_label); + do_ifelse (CCmode, NE, diff, const0_rtx, NULL_RTX, + diff_label, profile_probability::unlikely ()); if (library_call_label != NULL && bytes_is_const && bytes > max_bytes) { @@ -1317,7 +1325,7 @@ expand_compare_loop (rtx operands[]) loop with a branch to cleanup_label. */ emit_move_insn (target, const0_rtx); do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, - NULL_RTX, final_label); + NULL_RTX, final_label, profile_probability::unlikely ()); } rtx final_cleanup = gen_label_rtx (); @@ -1327,9 +1335,12 @@ expand_compare_loop (rtx operands[]) { /* If remainder length < word length, branch to final cleanup compare. */ + if (!bytes_is_const) - do_ifelse (CCmode, LT, cmp_rem, GEN_INT (load_mode_size), - NULL_RTX, final_cleanup); + { + do_ifelse (CCmode, LT, cmp_rem, GEN_INT (load_mode_size), + NULL_RTX, final_cleanup, profile_probability::even ()); + } /* load and compare 8B */ do_load_for_compare_from_addr (load_mode, d1_1, @@ -1354,7 +1365,7 @@ expand_compare_loop (rtx operands[]) } do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX, - dcond, diff_label); + dcond, diff_label, profile_probability::even ()); do_add3 (src1_addr, src1_addr, GEN_INT (load_mode_size)); do_add3 (src2_addr, src2_addr, GEN_INT (load_mode_size)); @@ -1365,9 +1376,8 @@ expand_compare_loop (rtx operands[]) else /* See if remaining length is now zero. We previously set target to 0 so we can just jump to the end. */ - do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, - NULL_RTX, final_label); - + do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, NULL_RTX, + final_label, profile_probability::unlikely ()); } /* Cases: @@ -1450,7 +1460,7 @@ expand_compare_loop (rtx operands[]) than one loop iteration, in which case go do the overlap load compare path. */ do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (loop_bytes), - NULL_RTX, nonconst_overlap); + NULL_RTX, nonconst_overlap, profile_probability::even ()); rtx rem4k = gen_reg_rtx (word_mode); rtx dist1 = gen_reg_rtx (word_mode); @@ -1460,12 +1470,14 @@ expand_compare_loop (rtx operands[]) emit_insn (gen_andsi3 (dist1, src1_addr, GEN_INT (0xfff))); else emit_insn (gen_anddi3 (dist1, src1_addr, GEN_INT (0xfff))); - do_ifelse (CCmode, LE, dist1, rem4k, NULL_RTX, handle4k_label); + do_ifelse (CCmode, LE, dist1, rem4k, NULL_RTX, + handle4k_label, profile_probability::very_unlikely ()); if (word_mode == SImode) emit_insn (gen_andsi3 (dist2, src2_addr, GEN_INT (0xfff))); else emit_insn (gen_anddi3 (dist2, src2_addr, GEN_INT (0xfff))); - do_ifelse (CCmode, LE, dist2, rem4k, NULL_RTX, handle4k_label); + do_ifelse (CCmode, LE, dist2, rem4k, NULL_RTX, + handle4k_label, profile_probability::very_unlikely ()); /* We don't have a 4k boundary to deal with, so do a load/shift/compare and jump to diff. */ @@ -1817,7 +1829,8 @@ expand_block_compare_gpr(unsigned HOST_WIDE_INT bytes, unsigned int base_align, rtx ne_rtx = gen_rtx_NE (VOIDmode, cr, const0_rtx); rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, ne_rtx, fin_ref, pc_rtx); - rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + add_reg_br_prob_note (j, profile_probability::unlikely ()); JUMP_LABEL (j) = final_label; LABEL_NUSES (final_label) += 1; } @@ -2095,7 +2108,8 @@ expand_strncmp_align_check (rtx strncmp_label, rtx src_addr, HOST_WIDE_INT bytes rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, lab_ref, pc_rtx); - rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + add_reg_br_prob_note (j, profile_probability::unlikely ()); JUMP_LABEL (j) = strncmp_label; LABEL_NUSES (strncmp_label) += 1; } @@ -2265,7 +2279,8 @@ expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare, rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, lab_ref, pc_rtx); - rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + add_reg_br_prob_note (j, profile_probability::unlikely ()); JUMP_LABEL (j) = final_move_label; LABEL_NUSES (final_move_label) += 1; @@ -2282,7 +2297,8 @@ expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare, rtx ifelse0 = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp0eq_rtx, lab_ref, pc_rtx); - rtx j0 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse0)); + rtx_insn *j0 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse0)); + add_reg_br_prob_note (j0, profile_probability::unlikely ()); JUMP_LABEL (j0) = final_move_label; LABEL_NUSES (final_move_label) += 1; } @@ -2325,7 +2341,8 @@ expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare, rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, lab_ref, pc_rtx); - rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse)); + add_reg_br_prob_note (j, profile_probability::unlikely ()); JUMP_LABEL (j) = dst_label; LABEL_NUSES (dst_label) += 1; }