(define_split
[(set (match_operand:MODEF 0 "sse_reg_operand")
(float:MODEF (match_operand:SWI48 1 "nonimmediate_operand")))]
- "TARGET_SSE_PARTIAL_REG_DEPENDENCY
+ "TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
&& optimize_function_for_speed_p (cfun)
- && epilogue_completed
&& (!EXT_REX_SSE_REG_P (operands[0])
|| TARGET_AVX512VL)"
[(set (match_dup 0)
[(set (match_operand:SF 0 "sse_reg_operand")
(float_truncate:SF
(match_operand:DF 1 "nonimmediate_operand")))]
- "TARGET_SSE_PARTIAL_REG_DEPENDENCY
+ "TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
&& optimize_function_for_speed_p (cfun)
- && epilogue_completed
&& (!REG_P (operands[1])
|| REGNO (operands[0]) != REGNO (operands[1]))
&& (!EXT_REX_SSE_REG_P (operands[0])
[(set (match_operand:DF 0 "sse_reg_operand")
(float_extend:DF
(match_operand:SF 1 "nonimmediate_operand")))]
- "TARGET_SSE_PARTIAL_REG_DEPENDENCY
+ "TARGET_SSE_PARTIAL_REG_DEPENDENCY && epilogue_completed
&& optimize_function_for_speed_p (cfun)
- && epilogue_completed
&& (!REG_P (operands[1])
|| REGNO (operands[0]) != REGNO (operands[1]))
&& (!EXT_REX_SSE_REG_P (operands[0])
ix86_expand_clear (operands[2]);
})
-; False dependency happens when destination is only updated by tzcnt,
-; lzcnt or popcnt. There is no false dependency when destination is
-; also used in source.
-(define_insn_and_split "*tzcnt<mode>_1_falsedep_1"
+(define_insn_and_split "*tzcnt<mode>_1"
[(set (reg:CCC FLAGS_REG)
(compare:CCC (match_operand:SWI48 1 "nonimmediate_operand" "rm")
(const_int 0)))
(set (match_operand:SWI48 0 "register_operand" "=r")
(ctz:SWI48 (match_dup 1)))]
- "TARGET_BMI
- && TARGET_AVOID_FALSE_DEP_FOR_BMI && optimize_function_for_speed_p (cfun)"
- "#"
- "&& reload_completed"
+ "TARGET_BMI"
+ "tzcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BMI && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
[(parallel
[(set (reg:CCC FLAGS_REG)
(compare:CCC (match_dup 1) (const_int 0)))
(set (match_dup 0)
(ctz:SWI48 (match_dup 1)))
(unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)])]
-{
- if (!reg_mentioned_p (operands[0], operands[1]))
- ix86_expand_clear (operands[0]);
-})
-
-(define_insn "*tzcnt<mode>_1_falsedep"
- [(set (reg:CCC FLAGS_REG)
- (compare:CCC (match_operand:SWI48 1 "nonimmediate_operand" "rm")
- (const_int 0)))
- (set (match_operand:SWI48 0 "register_operand" "=r")
- (ctz:SWI48 (match_dup 1)))
- (unspec [(match_operand:SWI48 2 "register_operand" "0")]
- UNSPEC_INSN_FALSE_DEP)]
- "TARGET_BMI"
- "tzcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
+ "ix86_expand_clear (operands[0]);"
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "prefix_rep" "1")
(set_attr "btver2_decode" "double")
(set_attr "mode" "<MODE>")])
-(define_insn "*tzcnt<mode>_1"
+; False dependency happens when destination is only updated by tzcnt,
+; lzcnt or popcnt. There is no false dependency when destination is
+; also used in source.
+(define_insn "*tzcnt<mode>_1_falsedep"
[(set (reg:CCC FLAGS_REG)
(compare:CCC (match_operand:SWI48 1 "nonimmediate_operand" "rm")
(const_int 0)))
(set (match_operand:SWI48 0 "register_operand" "=r")
- (ctz:SWI48 (match_dup 1)))]
+ (ctz:SWI48 (match_dup 1)))
+ (unspec [(match_operand:SWI48 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)]
"TARGET_BMI"
- "tzcnt{<imodesuffix>}\t{%1, %0|%0, %1}"
+ "tzcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
[(set_attr "type" "alu1")
(set_attr "prefix_0f" "1")
(set_attr "prefix_rep" "1")
(set_attr "znver1_decode" "vector")
(set_attr "mode" "<MODE>")])
-(define_expand "ctz<mode>2"
- [(parallel
- [(set (match_operand:SWI48 0 "register_operand")
- (ctz:SWI48
- (match_operand:SWI48 1 "nonimmediate_operand")))
- (clobber (reg:CC FLAGS_REG))])])
-
(define_insn_and_split "*ctzhi2"
[(set (match_operand:SI 0 "register_operand")
(ctz:SI
DONE;
})
-; False dependency happens when destination is only updated by tzcnt,
-; lzcnt or popcnt. There is no false dependency when destination is
-; also used in source.
-(define_insn_and_split "*ctz<mode>2_falsedep_1"
+(define_insn_and_split "ctz<mode>2"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(ctz:SWI48
(match_operand:SWI48 1 "nonimmediate_operand" "rm")))
(clobber (reg:CC FLAGS_REG))]
+ ""
+{
+ if (TARGET_BMI)
+ return "tzcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
+ else if (optimize_function_for_size_p (cfun))
+ ;
+ else if (TARGET_GENERIC)
+ /* tzcnt expands to 'rep bsf' and we can use it even if !TARGET_BMI. */
+ return "rep%; bsf{<imodesuffix>}\t{%1, %0|%0, %1}";
+
+ return "bsf{<imodesuffix>}\t{%1, %0|%0, %1}";
+}
"(TARGET_BMI || TARGET_GENERIC)
- && TARGET_AVOID_FALSE_DEP_FOR_BMI && optimize_function_for_speed_p (cfun)"
- "#"
- "&& reload_completed"
+ && TARGET_AVOID_FALSE_DEP_FOR_BMI && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
[(parallel
[(set (match_dup 0)
(ctz:SWI48 (match_dup 1)))
(unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
(clobber (reg:CC FLAGS_REG))])]
-{
- if (!reg_mentioned_p (operands[0], operands[1]))
- ix86_expand_clear (operands[0]);
-})
+ "ix86_expand_clear (operands[0]);"
+ [(set_attr "type" "alu1")
+ (set_attr "prefix_0f" "1")
+ (set (attr "prefix_rep")
+ (if_then_else
+ (ior (match_test "TARGET_BMI")
+ (and (not (match_test "optimize_function_for_size_p (cfun)"))
+ (match_test "TARGET_GENERIC")))
+ (const_string "1")
+ (const_string "0")))
+ (set_attr "mode" "<MODE>")])
+; False dependency happens when destination is only updated by tzcnt,
+; lzcnt or popcnt. There is no false dependency when destination is
+; also used in source.
(define_insn "*ctz<mode>2_falsedep"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(ctz:SWI48
(set_attr "prefix_rep" "1")
(set_attr "mode" "<MODE>")])
-(define_insn "*ctz<mode>2"
- [(set (match_operand:SWI48 0 "register_operand" "=r")
- (ctz:SWI48 (match_operand:SWI48 1 "nonimmediate_operand" "rm")))
- (clobber (reg:CC FLAGS_REG))]
- ""
-{
- if (TARGET_BMI)
- return "tzcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
- else if (optimize_function_for_size_p (cfun))
- ;
- else if (TARGET_GENERIC)
- /* tzcnt expands to 'rep bsf' and we can use it even if !TARGET_BMI. */
- return "rep%; bsf{<imodesuffix>}\t{%1, %0|%0, %1}";
-
- return "bsf{<imodesuffix>}\t{%1, %0|%0, %1}";
-}
- [(set_attr "type" "alu1")
- (set_attr "prefix_0f" "1")
- (set (attr "prefix_rep")
- (if_then_else
- (ior (match_test "TARGET_BMI")
- (and (not (match_test "optimize_function_for_size_p (cfun)"))
- (match_test "TARGET_GENERIC")))
- (const_string "1")
- (const_string "0")))
- (set_attr "mode" "<MODE>")])
-
(define_insn "bsr_rex64"
[(set (match_operand:DI 0 "register_operand" "=r")
(minus:DI (const_int 63)
DONE;
})
-; False dependency happens when destination is only updated by tzcnt,
-; lzcnt or popcnt. There is no false dependency when destination is
-; also used in source.
-(define_insn_and_split "*clz<mode>2_lzcnt_falsedep_1"
+(define_insn_and_split "clz<mode>2_lzcnt"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(clz:SWI48
(match_operand:SWI48 1 "nonimmediate_operand" "rm")))
(clobber (reg:CC FLAGS_REG))]
- "TARGET_LZCNT
- && TARGET_AVOID_FALSE_DEP_FOR_BMI && optimize_function_for_speed_p (cfun)"
- "#"
- "&& reload_completed"
+ "TARGET_LZCNT"
+ "lzcnt{<imodesuffix>}\t{%1, %0|%0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BMI && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
[(parallel
[(set (match_dup 0)
(clz:SWI48 (match_dup 1)))
(unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
(clobber (reg:CC FLAGS_REG))])]
-{
- if (!reg_mentioned_p (operands[0], operands[1]))
- ix86_expand_clear (operands[0]);
-})
-
-(define_insn "*clz<mode>2_lzcnt_falsedep"
- [(set (match_operand:SWI48 0 "register_operand" "=r")
- (clz:SWI48
- (match_operand:SWI48 1 "nonimmediate_operand" "rm")))
- (unspec [(match_operand:SWI48 2 "register_operand" "0")]
- UNSPEC_INSN_FALSE_DEP)
- (clobber (reg:CC FLAGS_REG))]
- "TARGET_LZCNT"
- "lzcnt{<imodesuffix>}\t{%1, %0|%0, %1}"
+ "ix86_expand_clear (operands[0]);"
[(set_attr "prefix_rep" "1")
(set_attr "type" "bitmanip")
(set_attr "mode" "<MODE>")])
-(define_insn "clz<mode>2_lzcnt"
+; False dependency happens when destination is only updated by tzcnt,
+; lzcnt or popcnt. There is no false dependency when destination is
+; also used in source.
+(define_insn "*clz<mode>2_lzcnt_falsedep"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(clz:SWI48
(match_operand:SWI48 1 "nonimmediate_operand" "rm")))
+ (unspec [(match_operand:SWI48 2 "register_operand" "0")]
+ UNSPEC_INSN_FALSE_DEP)
(clobber (reg:CC FLAGS_REG))]
"TARGET_LZCNT"
"lzcnt{<imodesuffix>}\t{%1, %0|%0, %1}"
;; Version of lzcnt/tzcnt that is expanded from intrinsics. This version
;; provides operand size as output when source operand is zero.
-(define_expand "<lt_zcnt>_<mode>"
- [(parallel
- [(set (match_operand:SWI248 0 "register_operand")
- (unspec:SWI248
- [(match_operand:SWI248 1 "nonimmediate_operand")] LT_ZCNT))
- (clobber (reg:CC FLAGS_REG))])])
-
-; False dependency happens when destination is only updated by tzcnt,
-; lzcnt or popcnt. There is no false dependency when destination is
-; also used in source.
-(define_insn_and_split "*<lt_zcnt>_<mode>_falsedep_1"
+(define_insn_and_split "<lt_zcnt>_<mode>"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(unspec:SWI48
[(match_operand:SWI48 1 "nonimmediate_operand" "rm")] LT_ZCNT))
(clobber (reg:CC FLAGS_REG))]
- "TARGET_AVOID_FALSE_DEP_FOR_BMI && optimize_function_for_speed_p (cfun)"
- "#"
- "&& reload_completed"
+ ""
+ "<lt_zcnt>{<imodesuffix>}\t{%1, %0|%0, %1}"
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BMI && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
[(parallel
[(set (match_dup 0)
(unspec:SWI48 [(match_dup 1)] LT_ZCNT))
(unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
(clobber (reg:CC FLAGS_REG))])]
-{
- if (!reg_mentioned_p (operands[0], operands[1]))
- ix86_expand_clear (operands[0]);
-})
+ "ix86_expand_clear (operands[0]);"
+ [(set_attr "type" "<lt_zcnt_type>")
+ (set_attr "prefix_0f" "1")
+ (set_attr "prefix_rep" "1")
+ (set_attr "mode" "<MODE>")])
+; False dependency happens when destination is only updated by tzcnt,
+; lzcnt or popcnt. There is no false dependency when destination is
+; also used in source.
(define_insn "*<lt_zcnt>_<mode>_falsedep"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(unspec:SWI48
(set_attr "prefix_rep" "1")
(set_attr "mode" "<MODE>")])
-(define_insn "*<lt_zcnt>_<mode>"
- [(set (match_operand:SWI248 0 "register_operand" "=r")
- (unspec:SWI248
- [(match_operand:SWI248 1 "nonimmediate_operand" "rm")] LT_ZCNT))
+(define_insn "<lt_zcnt>_hi"
+ [(set (match_operand:HI 0 "register_operand" "=r")
+ (unspec:HI
+ [(match_operand:HI 1 "nonimmediate_operand" "rm")] LT_ZCNT))
(clobber (reg:CC FLAGS_REG))]
""
- "<lt_zcnt>{<imodesuffix>}\t{%1, %0|%0, %1}"
+ "<lt_zcnt>{w}\t{%1, %0|%0, %1}"
[(set_attr "type" "<lt_zcnt_type>")
(set_attr "prefix_0f" "1")
(set_attr "prefix_rep" "1")
- (set_attr "mode" "<MODE>")])
+ (set_attr "mode" "HI")])
;; BMI instructions.
[(set_attr "type" "bitmanip")
(set_attr "mode" "<MODE>")])
-(define_expand "popcount<mode>2"
- [(parallel
- [(set (match_operand:SWI248 0 "register_operand")
- (popcount:SWI248
- (match_operand:SWI248 1 "nonimmediate_operand")))
- (clobber (reg:CC FLAGS_REG))])]
- "TARGET_POPCNT")
-
-(define_insn_and_split "*popcount<mode>2_falsedep_1"
+(define_insn_and_split "popcount<mode>2"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(popcount:SWI48
(match_operand:SWI48 1 "nonimmediate_operand" "rm")))
(clobber (reg:CC FLAGS_REG))]
- "TARGET_POPCNT
- && TARGET_AVOID_FALSE_DEP_FOR_BMI && optimize_function_for_speed_p (cfun)"
- "#"
- "&& reload_completed"
+ "TARGET_POPCNT"
+{
+#if TARGET_MACHO
+ return "popcnt\t{%1, %0|%0, %1}";
+#else
+ return "popcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
+#endif
+}
+ "&& TARGET_AVOID_FALSE_DEP_FOR_BMI && epilogue_completed
+ && optimize_function_for_speed_p (cfun)
+ && !reg_mentioned_p (operands[0], operands[1])"
[(parallel
[(set (match_dup 0)
(popcount:SWI48 (match_dup 1)))
(unspec [(match_dup 0)] UNSPEC_INSN_FALSE_DEP)
(clobber (reg:CC FLAGS_REG))])]
-{
- if (!reg_mentioned_p (operands[0], operands[1]))
- ix86_expand_clear (operands[0]);
-})
+ "ix86_expand_clear (operands[0]);"
+ [(set_attr "prefix_rep" "1")
+ (set_attr "type" "bitmanip")
+ (set_attr "mode" "<MODE>")])
+; False dependency happens when destination is only updated by tzcnt,
+; lzcnt or popcnt. There is no false dependency when destination is
+; also used in source.
(define_insn "*popcount<mode>2_falsedep"
[(set (match_operand:SWI48 0 "register_operand" "=r")
(popcount:SWI48
(set_attr "type" "bitmanip")
(set_attr "mode" "<MODE>")])
-(define_insn "*popcount<mode>2"
- [(set (match_operand:SWI248 0 "register_operand" "=r")
- (popcount:SWI248
- (match_operand:SWI248 1 "nonimmediate_operand" "rm")))
+(define_insn "popcounthi2"
+ [(set (match_operand:HI 0 "register_operand" "=r")
+ (popcount:HI
+ (match_operand:HI 1 "nonimmediate_operand" "rm")))
(clobber (reg:CC FLAGS_REG))]
"TARGET_POPCNT"
{
#if TARGET_MACHO
return "popcnt\t{%1, %0|%0, %1}";
#else
- return "popcnt{<imodesuffix>}\t{%1, %0|%0, %1}";
+ return "popcnt{w}\t{%1, %0|%0, %1}";
#endif
}
[(set_attr "prefix_rep" "1")
(set_attr "type" "bitmanip")
- (set_attr "mode" "<MODE>")])
+ (set_attr "mode" "HI")])
(define_expand "bswapdi2"
[(set (match_operand:DI 0 "register_operand")