Every translation block begins by loading cpu->neg.icount_decr.u32,
testing it and branching to the exit path. That is three host instructions
at the top of every TB. Blocks are short, so this is expensive: an emulated
alpha gcc 16.2.0 compiling a 255k line translation unit executes 34.2
billion TBs at 6.04 guest instructions each. Forcing CF_NOIRQ on for the
whole run, which is not correct but bounds the prize, is worth 12.0% of all
instructions retired.
The check does not have to be in every block. Interrupt latency is bounded
as long as every cycle in the guest control flow graph passes through at
least one block that polls. Any such cycle must contain either an edge
whose destination is at or below the start of the block it leaves from, or
an edge whose destination is not known at translation time: take the block
with the lowest start address in the cycle, and the edge entering it comes
from a block at or above it.
So record, during translation, whether this TB has such an edge.
translator_use_goto_tb() already sees every statically known destination,
and every target that emits goto_tb reaches it, so a backward edge sets
DisasContextBase::needs_exit_check there. Indirect destinations are flagged
by tcg_gen_lookup_and_goto_ptr(). Blocks with neither cannot close a cycle
on their own and can skip the poll.
The check is therefore emitted retroactively in gen_tb_end(), using the
same emit_before_op mechanism the can_do_io stores use, and only when one
of the two flags is set. icount opts out and keeps the unconditional
counter.
Measured on an x86-64 host, LTO build, on top of the preceding patches:
before: 869,178,598,378 instructions
after: 809,988,851,304 instructions -6.81%
before: 80.49s wall clock
after: 78.00s wall clock -3.10%
That is 57% of the 12.0% ceiling, which is about right: roughly a quarter of
TB exits are indirect and are still polled, plus every loop back edge.
For the series as a whole, against an unmodified LTO build, instructions
retired fall from 1,647,901,588,726 to 809,988,851,304 (-50.85%) and wall
clock from 133.57s to 78.00s (-41.61%). The two do not match because what
the series removes is mostly cheap, well-predicted dispatch overhead: IPC
falls from 2.53 to 2.12 as the remaining work gets less regular.
tests/tcg/alpha/test-xpage-chain.c still passes, the emulated compiler still
produces byte-identical output, and a tight loop under alarm(1) is still
interrupted, after 897 million iterations.
RFC because:
- The soundness argument depends on every goto_tb destination passing
through translator_use_goto_tb(). No target in the tree bypasses it
today, but nothing enforces that.
- System mode interrupt latency now depends on guest control flow rather
than on block count. The bound is one straight-line run between cycles,
which should be fine, but timer-driven guests deserve a closer look than
I can give them.
Signed-off-by: Matt Turner <[email protected]>
---
accel/tcg/translator.c | 57 ++++++++++++++++++++++++++++++++++++---
include/exec/translator.h | 2 ++
include/tcg/tcg.h | 2 ++
tcg/tcg-op.c | 2 ++
4 files changed, 60 insertions(+), 3 deletions(-)
diff --git ./accel/tcg/translator.c ./accel/tcg/translator.c
index 4921bf978c..ee61dec1c6 100644
--- ./accel/tcg/translator.c
+++ ./accel/tcg/translator.c
@@ -42,12 +42,29 @@ bool translator_io_start(DisasContextBase *db)
return true;
}
+/*
+ * Any cycle in the guest control flow graph must contain an edge whose
+ * destination is at or below the start of the block it leaves from, or an
+ * edge whose destination is not known at translation time. Only blocks with
+ * such an edge need the interrupt check, so defer the decision until the end
+ * of translation, when we know which edges this TB has.
+ *
+ * icount needs the counter unconditionally, so it opts out.
+ */
+static bool defer_exit_check(uint32_t cflags)
+{
+ return !(cflags & CF_USE_ICOUNT);
+}
+
static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
{
TCGv_i32 count = NULL;
TCGOp *icount_start_insn = NULL;
- if ((cflags & CF_USE_ICOUNT) || !(cflags & CF_NOIRQ)) {
+ tcg_ctx->exit_check_needed = false;
+
+ if ((cflags & CF_USE_ICOUNT) ||
+ (!(cflags & CF_NOIRQ) && !defer_exit_check(cflags))) {
count = tcg_temp_new_i32();
tcg_gen_ld_i32(count, tcg_env,
offsetof(CPUState, neg.icount_decr.u32) -
@@ -73,6 +90,12 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t
cflags)
*/
if (cflags & CF_NOIRQ) {
tcg_ctx->exitreq_label = NULL;
+ } else if (defer_exit_check(cflags)) {
+ /*
+ * Emitted retroactively by gen_tb_end(), but only if this TB can be
+ * part of a control flow cycle.
+ */
+ tcg_ctx->exitreq_label = gen_new_label();
} else {
tcg_ctx->exitreq_label = gen_new_label();
tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label);
@@ -88,7 +111,8 @@ static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t
cflags)
}
static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
- TCGOp *icount_start_insn, int num_insns)
+ TCGOp *icount_start_insn, int num_insns,
+ DisasContextBase *db, TCGOp *first_insn_start)
{
if (cflags & CF_USE_ICOUNT) {
/*
@@ -99,6 +123,23 @@ static void gen_tb_end(const TranslationBlock *tb, uint32_t
cflags,
tcgv_i32_arg(tcg_constant_i32(num_insns)));
}
+ if (tcg_ctx->exitreq_label && defer_exit_check(cflags) &&
+ !(cflags & CF_NOIRQ)) {
+ if (db->needs_exit_check || tcg_ctx->exit_check_needed) {
+ TCGv_i32 count = tcg_temp_new_i32();
+ TCGOp *save = tcg_ctx->emit_before_op;
+
+ tcg_ctx->emit_before_op = first_insn_start;
+ tcg_gen_ld_i32(count, tcg_env,
+ offsetof(CPUState, neg.icount_decr.u32) -
+ sizeof(CPUState));
+ tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label);
+ tcg_ctx->emit_before_op = save;
+ } else {
+ tcg_ctx->exitreq_label = NULL;
+ }
+ }
+
if (tcg_ctx->exitreq_label) {
gen_set_label(tcg_ctx->exitreq_label);
tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED);
@@ -117,6 +158,14 @@ bool translator_use_goto_tb(DisasContextBase *db, vaddr
dest)
return false;
}
+ /*
+ * A destination at or below the start of this TB can close a cycle, so
+ * this TB must poll for interrupts. See defer_exit_check().
+ */
+ if (dest <= db->pc_first) {
+ db->needs_exit_check = true;
+ }
+
#ifdef CONFIG_USER_ONLY
/*
* There are no page tables in user-only mode. Every mmap, mprotect and
@@ -153,6 +202,7 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb,
int *max_insns,
db->max_insns = *max_insns;
db->insn_start = NULL;
db->fake_insn = false;
+ db->needs_exit_check = false;
db->host_addr[0] = host_pc;
db->host_addr[1] = NULL;
db->record_start = 0;
@@ -219,7 +269,8 @@ void translator_loop(CPUState *cpu, TranslationBlock *tb,
int *max_insns,
/* Emit code to exit the TB, as indicated by db->is_jmp. */
ops->tb_stop(db, cpu);
- gen_tb_end(tb, cflags, icount_start_insn, db->num_insns);
+ gen_tb_end(tb, cflags, icount_start_insn, db->num_insns, db,
+ first_insn_start);
/*
* Manage can_do_io for the translation block: set to false before
diff --git ./include/exec/translator.h ./include/exec/translator.h
index 978dee25ad..003926c7f0 100644
--- ./include/exec/translator.h
+++ ./include/exec/translator.h
@@ -74,6 +74,8 @@ struct DisasContextBase {
int max_insns;
bool plugin_enabled;
bool fake_insn;
+ /* Set when this TB can be part of a control flow cycle. */
+ bool needs_exit_check;
uint8_t code_mmuidx;
struct TCGOp *insn_start;
void *host_addr[2];
diff --git ./include/tcg/tcg.h ./include/tcg/tcg.h
index 7669dc1c2d..be9ce7a0e2 100644
--- ./include/tcg/tcg.h
+++ ./include/tcg/tcg.h
@@ -389,6 +389,8 @@ struct TCGContext {
struct TCGLabelPoolData *pool_labels;
TCGLabel *exitreq_label;
+ /* Set by goto_ptr emission: destination is not known statically. */
+ bool exit_check_needed;
#ifdef CONFIG_PLUGIN
/*
diff --git ./tcg/tcg-op.c ./tcg/tcg-op.c
index a3efc56a9a..367e96627c 100644
--- ./tcg/tcg-op.c
+++ ./tcg/tcg-op.c
@@ -2616,6 +2616,7 @@ void tcg_gen_lookup_and_goto_ptr(void)
return;
}
+ tcg_ctx->exit_check_needed = true;
plugin_gen_disable_mem_helpers();
ptr = tcg_temp_ebb_new_ptr();
gen_helper_lookup_tb_ptr(ptr, tcg_env);
@@ -2642,6 +2643,7 @@ void tcg_gen_lookup_and_goto_ptr_inline(TCGv_i64 pc,
uint32_t flags,
return;
}
+ tcg_ctx->exit_check_needed = true;
plugin_gen_disable_mem_helpers();
QEMU_BUILD_BUG_ON(sizeof(((CPUJumpCache *)0)->array[0]) != 16);
--
2.54.0
