* [PATCH 0/4] Mirror map JIT memory for TCG
@ 2020-10-26 17:25 Joelle van Dyne
2020-10-26 17:25 ` [PATCH 1/4] tcg: add const hints for code pointers Joelle van Dyne
` (3 more replies)
0 siblings, 4 replies; 5+ messages in thread
From: Joelle van Dyne @ 2020-10-26 17:25 UTC (permalink / raw)
To: qemu-devel
To aid in debugging TCG, we enable a "mirror map" feature that, when enabled,
will use memfd_create() to create two mappings of the same memory for RW and RX.
The feature is only built when targeting system emulation on Linux and
configured with --enable-debug-tcg. It can be disabled with the runtime argument
`-accel tcg,mirror-jit=off`.
Although this patch only enables this feature for Linux, the main goal of this
is to enable, through a future patch, iOS JIT on older devices which require
strict W^X separation.
-j
Joelle van Dyne (4):
tcg: add const hints for code pointers
tcg: implement mirror mapped JIT for Linux
tcg: use mirror map JIT in code generation
tcg: make mirror mapped JIT optional
configure | 13 +++++
meson.build | 1 +
include/exec/exec-all.h | 8 +++
include/sysemu/tcg.h | 2 +-
include/tcg/tcg.h | 26 +++++++--
tcg/aarch64/tcg-target.h | 18 +++++-
tcg/arm/tcg-target.h | 15 ++++-
tcg/i386/tcg-target.h | 26 ++++++++-
tcg/mips/tcg-target.h | 12 +++-
tcg/ppc/tcg-target.h | 12 +++-
tcg/riscv/tcg-target.h | 13 ++++-
tcg/s390/tcg-target.h | 17 +++++-
tcg/sparc/tcg-target.h | 12 +++-
tcg/tci/tcg-target.h | 9 ++-
accel/tcg/cpu-exec.c | 7 ++-
accel/tcg/tcg-all.c | 31 +++++++++-
accel/tcg/translate-all.c | 106 +++++++++++++++++++++++++++++------
bsd-user/main.c | 2 +-
linux-user/main.c | 2 +-
tcg/tcg.c | 60 +++++++++++++-------
qemu-options.hx | 11 ++++
tcg/aarch64/tcg-target.c.inc | 48 +++++++++-------
tcg/arm/tcg-target.c.inc | 33 ++++++-----
tcg/i386/tcg-target.c.inc | 28 ++++-----
tcg/mips/tcg-target.c.inc | 64 ++++++++++++---------
tcg/ppc/tcg-target.c.inc | 57 ++++++++++++-------
tcg/riscv/tcg-target.c.inc | 51 +++++++++--------
tcg/s390/tcg-target.c.inc | 25 +++++----
tcg/sparc/tcg-target.c.inc | 33 ++++++-----
tcg/tcg-ldst.c.inc | 2 +-
tcg/tcg-pool.c.inc | 9 +--
tcg/tci/tcg-target.c.inc | 8 +--
32 files changed, 555 insertions(+), 206 deletions(-)
--
2.28.0
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH 1/4] tcg: add const hints for code pointers
2020-10-26 17:25 [PATCH 0/4] Mirror map JIT memory for TCG Joelle van Dyne
@ 2020-10-26 17:25 ` Joelle van Dyne
2020-10-26 17:25 ` [PATCH 2/4] tcg: implement mirror mapped JIT for Linux Joelle van Dyne
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Joelle van Dyne @ 2020-10-26 17:25 UTC (permalink / raw)
To: qemu-devel
Cc: Aleksandar Rikalo, open list:RISC-V TCG target, Stefan Weil,
Philippe Mathieu-Daudé,
Jiaxun Yang, open list:S390 TCG target,
open list:AArch64 TCG target, Palmer Dabbelt, Joelle van Dyne,
Huacai Chen, Alistair Francis, Aurelien Jarno, Richard Henderson
We will introduce mirror mapping for JIT segment with separate RX and RW
access. Adding 'const' hints will make it easier to identify read-only
accesses and allow us to easier catch bugs at compile time in the future.
Signed-off-by: Joelle van Dyne <j@getutm.app>
---
include/tcg/tcg.h | 8 ++++----
tcg/tcg.c | 4 ++--
tcg/aarch64/tcg-target.c.inc | 19 +++++++++++--------
tcg/arm/tcg-target.c.inc | 12 +++++++-----
tcg/i386/tcg-target.c.inc | 10 +++++-----
tcg/mips/tcg-target.c.inc | 33 +++++++++++++++++++--------------
tcg/ppc/tcg-target.c.inc | 23 ++++++++++++++---------
tcg/riscv/tcg-target.c.inc | 11 ++++++-----
tcg/s390/tcg-target.c.inc | 9 +++++----
tcg/sparc/tcg-target.c.inc | 10 +++++-----
tcg/tcg-ldst.c.inc | 2 +-
tcg/tci/tcg-target.c.inc | 2 +-
12 files changed, 80 insertions(+), 63 deletions(-)
diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index 8804a8c4a2..79c5ff8dab 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -261,7 +261,7 @@ struct TCGLabel {
unsigned refs : 16;
union {
uintptr_t value;
- tcg_insn_unit *value_ptr;
+ const tcg_insn_unit *value_ptr;
} u;
QSIMPLEQ_HEAD(, TCGRelocation) relocs;
QSIMPLEQ_ENTRY(TCGLabel) next;
@@ -593,7 +593,7 @@ struct TCGContext {
int nb_ops;
/* goto_tb support */
- tcg_insn_unit *code_buf;
+ const tcg_insn_unit *code_buf;
uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */
uintptr_t *tb_jmp_insn_offset; /* tb->jmp_target_arg if direct_jump */
uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_arg if !direct_jump */
@@ -1099,7 +1099,7 @@ static inline TCGLabel *arg_label(TCGArg i)
* correct result.
*/
-static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
+static inline ptrdiff_t tcg_ptr_byte_diff(const void *a, const void *b)
{
return a - b;
}
@@ -1113,7 +1113,7 @@ static inline ptrdiff_t tcg_ptr_byte_diff(void *a, void *b)
* to the destination address.
*/
-static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, void *target)
+static inline ptrdiff_t tcg_pcrel_diff(TCGContext *s, const void *target)
{
return tcg_ptr_byte_diff(target, s->code_ptr);
}
diff --git a/tcg/tcg.c b/tcg/tcg.c
index a8c28440e2..bb890c506d 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -148,7 +148,7 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
intptr_t arg2);
static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
TCGReg base, intptr_t ofs);
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
static int tcg_target_const_match(tcg_target_long val, TCGType type,
const TCGArgConstraint *arg_ct);
#ifdef TCG_TARGET_NEED_LDST_LABELS
@@ -295,7 +295,7 @@ static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
}
-static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
+static void tcg_out_label(TCGContext *s, TCGLabel *l, const tcg_insn_unit *ptr)
{
tcg_debug_assert(!l->has_value);
l->has_value = 1;
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index 26f71cb599..1aa5f37fc6 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -78,7 +78,8 @@ static const int tcg_target_call_oarg_regs[1] = {
#define TCG_REG_GUEST_BASE TCG_REG_X28
#endif
-static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc26(tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
if (offset == sextract64(offset, 0, 26)) {
@@ -90,7 +91,8 @@ static inline bool reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
return false;
}
-static inline bool reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc19(tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
if (offset == sextract64(offset, 0, 19)) {
@@ -1306,14 +1308,14 @@ static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
}
}
-static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target)
{
ptrdiff_t offset = target - s->code_ptr;
tcg_debug_assert(offset == sextract64(offset, 0, 26));
tcg_out_insn(s, 3206, B, offset);
}
-static inline void tcg_out_goto_long(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target)
{
ptrdiff_t offset = target - s->code_ptr;
if (offset == sextract64(offset, 0, 26)) {
@@ -1329,7 +1331,7 @@ static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
tcg_out_insn(s, 3207, BLR, reg);
}
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
{
ptrdiff_t offset = target - s->code_ptr;
if (offset == sextract64(offset, 0, 26)) {
@@ -1568,7 +1570,7 @@ static void * const qemu_st_helpers[16] = {
[MO_BEQ] = helper_be_stq_mmu,
};
-static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
+static inline void tcg_out_adr(TCGContext *s, TCGReg rd, const void *target)
{
ptrdiff_t offset = tcg_pcrel_diff(s, target);
tcg_debug_assert(offset == sextract64(offset, 0, 21));
@@ -1622,7 +1624,8 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
TCGType ext, TCGReg data_reg, TCGReg addr_reg,
- tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
+ const tcg_insn_unit *raddr,
+ tcg_insn_unit *label_ptr)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
@@ -1849,7 +1852,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
#endif /* CONFIG_SOFTMMU */
}
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg args[TCG_MAX_OP_ARGS],
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index 62c37a954b..5ceb06a971 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -187,7 +187,8 @@ static const uint8_t tcg_cond_to_arm_cond[] = {
[TCG_COND_GTU] = COND_HI,
};
-static inline bool reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc24(tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
if (offset == sextract32(offset, 0, 24)) {
@@ -197,7 +198,8 @@ static inline bool reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
return false;
}
-static inline bool reloc_pc13(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static inline bool reloc_pc13(tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
ptrdiff_t offset = tcg_ptr_byte_diff(target, code_ptr) - 8;
@@ -1019,7 +1021,7 @@ static inline void tcg_out_st8(TCGContext *s, int cond,
* with the code buffer limited to 16MB we wouldn't need the long case.
* But we also use it for the tail-call to the qemu_ld/st helpers, which does.
*/
-static void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr)
+static void tcg_out_goto(TCGContext *s, int cond, const tcg_insn_unit *addr)
{
intptr_t addri = (intptr_t)addr;
ptrdiff_t disp = tcg_pcrel_diff(s, addr);
@@ -1033,7 +1035,7 @@ static void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr)
/* The call case is mostly used for helpers - so it's not unreasonable
* for them to be beyond branch range */
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr)
{
intptr_t addri = (intptr_t)addr;
ptrdiff_t disp = tcg_pcrel_diff(s, addr);
@@ -1326,7 +1328,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
helper code. */
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
TCGReg datalo, TCGReg datahi, TCGReg addrlo,
- TCGReg addrhi, tcg_insn_unit *raddr,
+ TCGReg addrhi, const tcg_insn_unit *raddr,
tcg_insn_unit *label_ptr)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index d8797ed398..69acb51f5d 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -165,7 +165,7 @@ static bool have_lzcnt;
# define have_lzcnt 0
#endif
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
@@ -1591,7 +1591,7 @@ static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1,
}
}
-static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest)
+static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest)
{
intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
@@ -1610,12 +1610,12 @@ static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest)
}
}
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
{
tcg_out_branch(s, 1, dest);
}
-static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest)
+static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest)
{
tcg_out_branch(s, 0, dest);
}
@@ -1774,7 +1774,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, bool is_64,
TCGMemOpIdx oi,
TCGReg datalo, TCGReg datahi,
TCGReg addrlo, TCGReg addrhi,
- tcg_insn_unit *raddr,
+ const tcg_insn_unit *raddr,
tcg_insn_unit **label_ptr)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index 41be574e89..ace413afde 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -139,12 +139,13 @@ static const TCGReg tcg_target_call_oarg_regs[2] = {
TCG_REG_V1
};
-static tcg_insn_unit *tb_ret_addr;
-static tcg_insn_unit *bswap32_addr;
-static tcg_insn_unit *bswap32u_addr;
-static tcg_insn_unit *bswap64_addr;
+static const tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *bswap32_addr;
+static const tcg_insn_unit *bswap32u_addr;
+static const tcg_insn_unit *bswap64_addr;
-static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static inline uint32_t reloc_pc16_val(const tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
/* Let the compiler perform the right-shift as part of the arithmetic. */
ptrdiff_t disp = target - (pc + 1);
@@ -152,18 +153,21 @@ static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target)
return disp & 0xffff;
}
-static inline void reloc_pc16(tcg_insn_unit *pc, tcg_insn_unit *target)
+static inline void reloc_pc16(tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
*pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target));
}
-static inline uint32_t reloc_26_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static inline uint32_t reloc_26_val(const tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
tcg_debug_assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0);
return ((uintptr_t)target >> 2) & 0x3ffffff;
}
-static inline void reloc_26(tcg_insn_unit *pc, tcg_insn_unit *target)
+static inline void reloc_26(tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
*pc = deposit32(*pc, 0, 26, reloc_26_val(pc, target));
}
@@ -516,7 +520,7 @@ static void tcg_out_opc_sa64(TCGContext *s, MIPSInsn opc1, MIPSInsn opc2,
* Type jump.
* Returns true if the branch was in range and the insn was emitted.
*/
-static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target)
+static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, const void *target)
{
uintptr_t dest = (uintptr_t)target;
uintptr_t from = (uintptr_t)s->code_ptr + 4;
@@ -631,7 +635,7 @@ static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
}
}
-static void tcg_out_bswap_subr(TCGContext *s, tcg_insn_unit *sub)
+static void tcg_out_bswap_subr(TCGContext *s, const tcg_insn_unit *sub)
{
bool ok = tcg_out_opc_jmp(s, OPC_JAL, sub);
tcg_debug_assert(ok);
@@ -1079,7 +1083,7 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
}
}
-static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
+static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail)
{
/* Note that the ABI requires the called function's address to be
loaded into T9, even if a direct branch is in range. */
@@ -1097,7 +1101,7 @@ static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
{
tcg_out_call_int(s, arg, false);
tcg_out_nop(s);
@@ -1289,7 +1293,8 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
TCGType ext,
TCGReg datalo, TCGReg datahi,
TCGReg addrlo, TCGReg addrhi,
- void *raddr, tcg_insn_unit *label_ptr[2])
+ const tcg_insn_unit *raddr,
+ tcg_insn_unit *label_ptr[2])
{
TCGLabelQemuLdst *label = new_ldst_label(s);
@@ -2430,7 +2435,7 @@ static void tcg_target_detect_isa(void)
sigaction(SIGILL, &sa_old, NULL);
}
-static tcg_insn_unit *align_code_ptr(TCGContext *s)
+static const tcg_insn_unit *align_code_ptr(TCGContext *s)
{
uintptr_t p = (uintptr_t)s->code_ptr;
if (p & 15) {
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index 18ee989f95..945ff2fba1 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -62,7 +62,7 @@
#define TCG_CT_CONST_MONE 0x2000
#define TCG_CT_CONST_WSZ 0x4000
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
TCGPowerISA have_isa;
static bool have_isel;
@@ -184,14 +184,16 @@ static inline bool in_range_b(tcg_target_long target)
return target == sextract64(target, 0, 26);
}
-static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static uint32_t reloc_pc24_val(const tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
tcg_debug_assert(in_range_b(disp));
return disp & 0x3fffffc;
}
-static bool reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
+static bool reloc_pc24(tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
if (in_range_b(disp)) {
@@ -201,14 +203,16 @@ static bool reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
return false;
}
-static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+static uint16_t reloc_pc14_val(const tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
tcg_debug_assert(disp == (int16_t) disp);
return disp & 0xfffc;
}
-static bool reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target)
+static bool reloc_pc14(tcg_insn_unit *pc,
+ const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
if (disp == (int16_t) disp) {
@@ -1106,7 +1110,7 @@ static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
tcg_out_zori32(s, dst, src, c, XORI, XORIS);
}
-static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target)
+static void tcg_out_b(TCGContext *s, int mask, const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_pcrel_diff(s, target);
if (in_range_b(disp)) {
@@ -1762,7 +1766,7 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
{
#ifdef _CALL_AIX
/* Look through the descriptor. If the branch is in range, and we
@@ -1987,7 +1991,8 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, MemOp opc,
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
TCGReg datalo_reg, TCGReg datahi_reg,
TCGReg addrlo_reg, TCGReg addrhi_reg,
- tcg_insn_unit *raddr, tcg_insn_unit *lptr)
+ const tcg_insn_unit *raddr,
+ tcg_insn_unit *lptr)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
@@ -2341,7 +2346,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out32(s, BCCTR | BO_ALWAYS);
/* Epilogue */
- s->code_gen_epilogue = tb_ret_addr = s->code_ptr;
+ tb_ret_addr = s->code_gen_epilogue = s->code_ptr;
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index d536f3ccc1..5432ee89db 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -854,14 +854,14 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
g_assert_not_reached();
}
-static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
+static inline void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target)
{
ptrdiff_t offset = tcg_pcrel_diff(s, target);
tcg_debug_assert(offset == sextreg(offset, 1, 20) << 1);
tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, offset);
}
-static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
+static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail)
{
TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA;
ptrdiff_t offset = tcg_pcrel_diff(s, arg);
@@ -888,7 +888,7 @@ static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
{
tcg_out_call_int(s, arg, false);
}
@@ -1022,7 +1022,8 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
TCGType ext,
TCGReg datalo, TCGReg datahi,
TCGReg addrlo, TCGReg addrhi,
- void *raddr, tcg_insn_unit **label_ptr)
+ const tcg_insn_unit *raddr,
+ tcg_insn_unit **label_ptr)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
@@ -1274,7 +1275,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
#endif
}
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
static void tcg_out_op(TCGContext *s, TCGOpcode opc,
const TCGArg *args, const int *const_args)
diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc
index c5e096449b..7c6917f3c4 100644
--- a/tcg/s390/tcg-target.c.inc
+++ b/tcg/s390/tcg-target.c.inc
@@ -363,7 +363,7 @@ static void * const qemu_st_helpers[16] = {
};
#endif
-static tcg_insn_unit *tb_ret_addr;
+static const tcg_insn_unit *tb_ret_addr;
uint64_t s390_facilities;
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
@@ -1302,7 +1302,7 @@ static void tgen_extract(TCGContext *s, TCGReg dest, TCGReg src,
tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
}
-static void tgen_gotoi(TCGContext *s, int cc, tcg_insn_unit *dest)
+static void tgen_gotoi(TCGContext *s, int cc, const tcg_insn_unit *dest)
{
ptrdiff_t off = dest - s->code_ptr;
if (off == (int16_t)off) {
@@ -1415,7 +1415,7 @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
tgen_branch(s, cc, l);
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
{
ptrdiff_t off = dest - s->code_ptr;
if (off == (int32_t)off) {
@@ -1593,7 +1593,8 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
TCGReg data, TCGReg addr,
- tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
+ const tcg_insn_unit *raddr,
+ tcg_insn_unit *label_ptr)
{
TCGLabelQemuLdst *label = new_ldst_label(s);
diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc
index 6775bd30fc..5eac18b719 100644
--- a/tcg/sparc/tcg-target.c.inc
+++ b/tcg/sparc/tcg-target.c.inc
@@ -840,7 +840,7 @@ static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh,
tcg_out_mov(s, TCG_TYPE_I64, rl, tmp);
}
-static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest,
+static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest,
bool in_prologue)
{
ptrdiff_t disp = tcg_pcrel_diff(s, dest);
@@ -855,7 +855,7 @@ static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest,
}
}
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest)
{
tcg_out_call_nodelay(s, dest, false);
tcg_out_nop(s);
@@ -868,8 +868,8 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
}
#ifdef CONFIG_SOFTMMU
-static tcg_insn_unit *qemu_ld_trampoline[16];
-static tcg_insn_unit *qemu_st_trampoline[16];
+static const tcg_insn_unit *qemu_ld_trampoline[16];
+static const tcg_insn_unit *qemu_st_trampoline[16];
static void emit_extend(TCGContext *s, TCGReg r, int op)
{
@@ -1163,7 +1163,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
#ifdef CONFIG_SOFTMMU
unsigned memi = get_mmuidx(oi);
TCGReg addrz, param;
- tcg_insn_unit *func;
+ const tcg_insn_unit *func;
tcg_insn_unit *label_ptr;
addrz = tcg_out_tlb_load(s, addr, memi, memop,
diff --git a/tcg/tcg-ldst.c.inc b/tcg/tcg-ldst.c.inc
index 05f9b3ccd6..eaba08700e 100644
--- a/tcg/tcg-ldst.c.inc
+++ b/tcg/tcg-ldst.c.inc
@@ -28,7 +28,7 @@ typedef struct TCGLabelQemuLdst {
TCGReg addrhi_reg; /* reg index for high word of guest virtual addr */
TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
- tcg_insn_unit *raddr; /* gen code addr of the next IR of qemu_ld/st IR */
+ const tcg_insn_unit *raddr; /* gen code addr of the next IR of qemu_ld/st */
tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
} TCGLabelQemuLdst;
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index 231b9b1775..d5a4d9d37c 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -545,7 +545,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type,
old_code_ptr[1] = s->code_ptr - old_code_ptr;
}
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg)
{
uint8_t *old_code_ptr = s->code_ptr;
tcg_out_op_t(s, INDEX_op_call);
--
2.28.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 2/4] tcg: implement mirror mapped JIT for Linux
2020-10-26 17:25 [PATCH 0/4] Mirror map JIT memory for TCG Joelle van Dyne
2020-10-26 17:25 ` [PATCH 1/4] tcg: add const hints for code pointers Joelle van Dyne
@ 2020-10-26 17:25 ` Joelle van Dyne
2020-10-26 17:25 ` [PATCH 3/4] tcg: use mirror map JIT in code generation Joelle van Dyne
2020-10-26 17:25 ` [PATCH 4/4] tcg: make mirror mapped JIT optional Joelle van Dyne
3 siblings, 0 replies; 5+ messages in thread
From: Joelle van Dyne @ 2020-10-26 17:25 UTC (permalink / raw)
To: qemu-devel; +Cc: Paolo Bonzini, Joelle van Dyne, Richard Henderson
When configured with --enable-debug-tcg on Linux, we enable JIT mirror map
which separates the JIT allocation into a RX-only address and a RW-only
address. The backing is a memfd_create() object.
This allows us to catch bugs where we unintentionally write to the JIT
region. However, the main use case will be for other platforms that require
strict W^X separation to run.
Signed-off-by: Joelle van Dyne <j@getutm.app>
---
configure | 13 ++++++++
meson.build | 1 +
include/exec/exec-all.h | 8 +++++
include/tcg/tcg.h | 18 ++++++++++
accel/tcg/cpu-exec.c | 7 +++-
accel/tcg/translate-all.c | 69 +++++++++++++++++++++++++++++++++++----
6 files changed, 108 insertions(+), 8 deletions(-)
diff --git a/configure b/configure
index e6754c1e87..9a632821c4 100755
--- a/configure
+++ b/configure
@@ -448,6 +448,7 @@ meson=""
ninja=""
skip_meson=no
gettext=""
+mirror_jit="no"
bogus_os="no"
malloc_trim="auto"
@@ -5725,6 +5726,14 @@ if test "$mingw32" = "yes" ; then
done
fi
+# Enable mirror mapping if debug_tcg on supported platform
+
+if test "$debug_tcg" = "yes"; then
+ if test "$linux" = "yes"; then
+ mirror_jit=yes
+ fi
+fi
+
qemu_confdir="$sysconfdir/$qemu_suffix"
qemu_moddir="$libdir/$qemu_suffix"
qemu_datadir="$datadir/$qemu_suffix"
@@ -6843,6 +6852,10 @@ if test "$safe_stack" = "yes"; then
echo "CONFIG_SAFESTACK=y" >> $config_host_mak
fi
+if test "$mirror_jit" = "yes"; then
+ echo "CONFIG_MIRROR_JIT=y" >> $config_host_mak
+fi
+
# If we're using a separate build tree, set it up now.
# DIRS are directories which we simply mkdir in the build tree;
# LINKS are things to symlink back into the source tree
diff --git a/meson.build b/meson.build
index 7627a0ae46..9630edff9f 100644
--- a/meson.build
+++ b/meson.build
@@ -2114,6 +2114,7 @@ endif
summary_info += {'thread sanitizer': config_host.has_key('CONFIG_TSAN')}
summary_info += {'rng-none': config_host.has_key('CONFIG_RNG_NONE')}
summary_info += {'Linux keyring': config_host.has_key('CONFIG_SECRET_KEYRING')}
+summary_info += {'mirror JIT': config_host.has_key('CONFIG_MIRROR_JIT')}
summary(summary_info, bool_yn: true)
if not supported_cpus.contains(cpu)
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 4707ac140c..921767a51b 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -519,6 +519,14 @@ struct TranslationBlock {
uintptr_t jmp_list_head;
uintptr_t jmp_list_next[2];
uintptr_t jmp_dest[2];
+
+#if defined(CONFIG_MIRROR_JIT)
+ /*
+ * Store difference to writable mirror
+ * We need this when patching the jump instructions
+ */
+ ptrdiff_t code_rw_mirror_diff;
+#endif
};
extern bool parallel_cpus;
diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index 79c5ff8dab..a76c259d1b 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -627,6 +627,10 @@ struct TCGContext {
size_t code_gen_buffer_size;
void *code_gen_ptr;
void *data_gen_ptr;
+#if defined(CONFIG_MIRROR_JIT)
+ int code_gen_buffer_fd;
+ ptrdiff_t code_rw_mirror_diff;
+#endif
/* Threshold to flush the translated code buffer. */
void *code_gen_highwater;
@@ -677,6 +681,20 @@ struct TCGContext {
target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
};
+static inline void *tcg_mirror_ptr_rw(TCGContext *s, const void *rx)
+{
+#if defined(CONFIG_MIRROR_JIT)
+ return (void *)rx + s->code_rw_mirror_diff;
+#else
+ return (void *)rx;
+#endif
+}
+
+static inline tcg_insn_unit *tcg_code_ptr_rw(TCGContext *s, const void *rx)
+{
+ return (tcg_insn_unit *)tcg_mirror_ptr_rw(s, rx);
+}
+
extern TCGContext tcg_init_ctx;
extern __thread TCGContext *tcg_ctx;
extern TCGv_env cpu_env;
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index 58aea605d8..f7eb6fcd77 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -354,7 +354,12 @@ void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
if (TCG_TARGET_HAS_direct_jump) {
uintptr_t offset = tb->jmp_target_arg[n];
uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr;
- tb_target_set_jmp_target(tc_ptr, tc_ptr + offset, addr);
+#if defined(CONFIG_MIRROR_JIT)
+ uintptr_t wr_addr = tc_ptr + offset + tb->code_rw_mirror_diff;
+#else
+ uintptr_t wr_addr = tc_ptr + offset;
+#endif
+ tb_target_set_jmp_target(tc_ptr, tc_ptr + offset, addr, wr_addr);
} else {
tb->jmp_target_arg[n] = addr;
}
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index d76097296d..bfb856ac1a 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -59,6 +59,7 @@
#include "sysemu/cpus.h"
#include "sysemu/cpu-timers.h"
#include "sysemu/tcg.h"
+#include "qemu/memfd.h"
/* #define DEBUG_TB_INVALIDATE */
/* #define DEBUG_TB_FLUSH */
@@ -302,10 +303,12 @@ static target_long decode_sleb128(uint8_t **pp)
static int encode_search(TranslationBlock *tb, uint8_t *block)
{
- uint8_t *highwater = tcg_ctx->code_gen_highwater;
- uint8_t *p = block;
+ const uint8_t *highwater, *start;
+ uint8_t *p;
int i, j, n;
+ highwater = tcg_mirror_ptr_rw(tcg_ctx, tcg_ctx->code_gen_highwater);
+ start = p = tcg_mirror_ptr_rw(tcg_ctx, block);
for (i = 0, n = tb->icount; i < n; ++i) {
target_ulong prev;
@@ -329,7 +332,7 @@ static int encode_search(TranslationBlock *tb, uint8_t *block)
}
}
- return p - block;
+ return p - start;
}
/* The cpu state corresponding to 'searched_pc' is restored.
@@ -1067,12 +1070,29 @@ static inline void *alloc_code_gen_buffer(void)
#else
static inline void *alloc_code_gen_buffer(void)
{
- int prot = PROT_WRITE | PROT_READ | PROT_EXEC;
- int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+ int prot = PROT_READ | PROT_EXEC;
+ int flags = 0;
size_t size = tcg_ctx->code_gen_buffer_size;
+ int fd = -1;
void *buf;
- buf = mmap(NULL, size, prot, flags, -1, 0);
+#if defined(CONFIG_MIRROR_JIT)
+#if defined(CONFIG_LINUX)
+ fd = qemu_memfd_create("tcg-jit", size, false, 0, 0, NULL);
+ if (fd < 0) {
+ return NULL;
+ }
+ tcg_ctx->code_gen_buffer_fd = fd;
+ flags |= MAP_SHARED;
+#else /* defined(CONFIG_LINUX) */
+#error "Mirror JIT unimplemented for this platform."
+#endif /* defined(CONFIG_LINUX) */
+#else /* defined(CONFIG_MIRROR_JIT) */
+ prot |= PROT_WRITE;
+ flags |= MAP_ANONYMOUS | MAP_PRIVATE;
+#endif /* defined(CONFIG_MIRROR_JIT) */
+
+ buf = mmap(NULL, size, prot, flags, fd, 0);
if (buf == MAP_FAILED) {
return NULL;
}
@@ -1118,7 +1138,28 @@ static inline void *alloc_code_gen_buffer(void)
}
#endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
-static inline void code_gen_alloc(size_t tb_size)
+#if defined(CONFIG_MIRROR_JIT)
+#if defined(CONFIG_LINUX)
+static inline void *alloc_jit_rw_mirror(void)
+{
+ int fd = tcg_ctx->code_gen_buffer_fd;
+ size_t size = tcg_ctx->code_gen_buffer_size;
+ void *buf;
+
+ assert(fd >= 0);
+ buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (buf == MAP_FAILED) {
+ return NULL;
+ }
+
+ return buf;
+}
+#else
+#error "Mirror JIT unimplemented for this platform."
+#endif /* CONFIG_IOS */
+#endif /* CONFIG_MIRROR_JIT */
+
+static inline void code_gen_alloc(size_t tb_size, bool mirror_jit)
{
tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size);
tcg_ctx->code_gen_buffer = alloc_code_gen_buffer();
@@ -1126,6 +1167,17 @@ static inline void code_gen_alloc(size_t tb_size)
fprintf(stderr, "Could not allocate dynamic translator buffer\n");
exit(1);
}
+#if defined(CONFIG_MIRROR_JIT)
+ void *mirror;
+
+ /* For platforms that need a mirror mapping for code execution */
+ mirror = alloc_jit_rw_mirror();
+ if (mirror == NULL) {
+ fprintf(stderr, "Could not remap code buffer mirror\n");
+ exit(1);
+ }
+ tcg_ctx->code_rw_mirror_diff = mirror - tcg_ctx->code_gen_buffer;
+#endif /* CONFIG_MIRROR_JIT */
}
static bool tb_cmp(const void *ap, const void *bp)
@@ -1721,6 +1773,9 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
cpu_loop_exit(cpu);
}
+#if defined(CONFIG_MIRROR_JIT)
+ tb->code_rw_mirror_diff = tcg_ctx->code_rw_mirror_diff;
+#endif
gen_code_buf = tcg_ctx->code_gen_ptr;
tb->tc.ptr = gen_code_buf;
tb->pc = pc;
--
2.28.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 3/4] tcg: use mirror map JIT in code generation
2020-10-26 17:25 [PATCH 0/4] Mirror map JIT memory for TCG Joelle van Dyne
2020-10-26 17:25 ` [PATCH 1/4] tcg: add const hints for code pointers Joelle van Dyne
2020-10-26 17:25 ` [PATCH 2/4] tcg: implement mirror mapped JIT for Linux Joelle van Dyne
@ 2020-10-26 17:25 ` Joelle van Dyne
2020-10-26 17:25 ` [PATCH 4/4] tcg: make mirror mapped JIT optional Joelle van Dyne
3 siblings, 0 replies; 5+ messages in thread
From: Joelle van Dyne @ 2020-10-26 17:25 UTC (permalink / raw)
To: qemu-devel
Cc: Aleksandar Rikalo, open list:RISC-V TCG target, Stefan Weil,
Richard Henderson, Philippe Mathieu-Daudé,
Jiaxun Yang, open list:S390 TCG target,
open list:AArch64 TCG target, Palmer Dabbelt, Joelle van Dyne,
Huacai Chen, Alistair Francis, Aurelien Jarno
In TCG code generation, there are places where we need to mutate code in JIT
memory. In the case of a mirror mapped implementation, we need to get the
RW pointer to perform any writes.
To ensure cache coherency, we flush the data cache in the RW mapping and
then invalidate the instruction cache in the RX mapping (where applicable).
Cache flushing is defined per-platform. GCC and Apple Clang are supported.
Signed-off-by: Joelle van Dyne <j@getutm.app>
---
tcg/aarch64/tcg-target.h | 18 +++++++++++-
tcg/arm/tcg-target.h | 15 +++++++++-
tcg/i386/tcg-target.h | 26 +++++++++++++++--
tcg/mips/tcg-target.h | 12 +++++++-
tcg/ppc/tcg-target.h | 12 +++++++-
tcg/riscv/tcg-target.h | 13 ++++++++-
tcg/s390/tcg-target.h | 17 ++++++++++-
tcg/sparc/tcg-target.h | 12 +++++++-
tcg/tci/tcg-target.h | 9 +++++-
tcg/tcg.c | 56 ++++++++++++++++++++++++------------
tcg/aarch64/tcg-target.c.inc | 33 ++++++++++++---------
tcg/arm/tcg-target.c.inc | 25 +++++++++-------
tcg/i386/tcg-target.c.inc | 18 ++++++------
tcg/mips/tcg-target.c.inc | 35 +++++++++++++---------
tcg/ppc/tcg-target.c.inc | 38 +++++++++++++++---------
tcg/riscv/tcg-target.c.inc | 40 +++++++++++++++-----------
tcg/s390/tcg-target.c.inc | 16 +++++------
tcg/sparc/tcg-target.c.inc | 23 +++++++++------
tcg/tcg-pool.c.inc | 9 +++---
tcg/tci/tcg-target.c.inc | 6 ++--
20 files changed, 303 insertions(+), 130 deletions(-)
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index 663dd0b95e..f605257ed5 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -148,12 +148,28 @@ typedef enum {
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
+#if defined(__APPLE__)
+void sys_dcache_flush(void *start, size_t len);
+#endif
+
static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
{
__builtin___clear_cache((char *)start, (char *)stop);
}
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__APPLE__)
+ sys_dcache_flush((char *)start, stop - start);
+#elif defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index 17e771374d..5862e28200 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -139,8 +139,21 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
__builtin___clear_cache((char *) start, (char *) stop);
}
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__APPLE__)
+ sys_dcache_flush((char *)start, stop - start);
+#elif defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
+
/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index abd4ac7fc0..1ff5d30542 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -206,16 +206,38 @@ extern bool have_avx2;
#define TCG_TARGET_extract_i64_valid(ofs, len) \
(((ofs) == 8 && (len) == 8) || ((ofs) + (len)) == 32)
+#if defined(__APPLE__)
+void sys_dcache_flush(void *start, size_t len);
+#endif
+
static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
{
}
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__APPLE__)
+ sys_dcache_flush((char *)start, stop - start);
+#elif defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
+
static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
- uintptr_t jmp_addr, uintptr_t addr)
+ uintptr_t jmp_addr, uintptr_t addr,
+ uintptr_t wr_addr)
{
/* patch the branch destination */
- qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
+ qatomic_set((int32_t *)wr_addr, addr - (jmp_addr + 4));
/* no need to flush icache explicitly */
+#if defined(CONFIG_MIRROR_JIT)
+ /* we do need to flush mirror dcache */
+ flush_dcache_range(wr_addr, wr_addr + 4);
+#endif
}
/* This defines the natural memory order supported by this
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index c6b091d849..74e9d8ee92 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -212,7 +212,17 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
cacheflush ((void *)start, stop-start, ICACHE);
}
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
#ifdef CONFIG_SOFTMMU
#define TCG_TARGET_NEED_LDST_LABELS
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index be10363956..5b65628333 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -176,7 +176,17 @@ extern bool have_vsx;
#define TCG_TARGET_HAS_cmpsel_vec 0
void flush_icache_range(uintptr_t start, uintptr_t stop);
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
#define TCG_TARGET_DEFAULT_MO (0)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h
index 032439d806..64052abded 100644
--- a/tcg/riscv/tcg-target.h
+++ b/tcg/riscv/tcg-target.h
@@ -164,8 +164,19 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
__builtin___clear_cache((char *)start, (char *)stop);
}
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
+
/* not defined -- call should be eliminated at compile time */
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
#define TCG_TARGET_DEFAULT_MO (0)
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index 63c8797bd3..60aa7934e1 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -149,13 +149,28 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
{
}
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
+
static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
- uintptr_t jmp_addr, uintptr_t addr)
+ uintptr_t jmp_addr, uintptr_t addr,
+ uintptr_t wr_addr)
{
/* patch the branch destination */
intptr_t disp = addr - (jmp_addr - 2);
qatomic_set((int32_t *)jmp_addr, disp / 2);
/* no need to flush icache explicitly */
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 4);
+#endif
}
#ifdef CONFIG_SOFTMMU
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index 633841ebf2..b22e59a512 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -176,7 +176,17 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
}
}
-void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t);
+void tb_target_set_jmp_target(uintptr_t, uintptr_t, uintptr_t, uintptr_t);
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+#if defined(__GNUC__)
+ __builtin___clear_cache((char *)start, (char *)stop);
+#else
+#error "Missing function to flush data cache"
+#endif
+}
+#endif
#define TCG_TARGET_NEED_POOL_LABELS
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index 8c1c1d265d..4e8070aae4 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -195,6 +195,12 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
{
}
+#if defined(CONFIG_MIRROR_JIT)
+static inline void flush_dcache_range(uintptr_t start, uintptr_t stop)
+{
+}
+#endif
+
/* We could notice __i386__ or __s390x__ and reduce the barriers depending
on the host. But if you want performance, you use the normal backend.
We prefer consistency across hosts on this. */
@@ -203,7 +209,8 @@ static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
#define TCG_TARGET_HAS_MEMORY_BSWAP 1
static inline void tb_target_set_jmp_target(uintptr_t tc_ptr,
- uintptr_t jmp_addr, uintptr_t addr)
+ uintptr_t jmp_addr, uintptr_t addr,
+ uintptr_t wr_addr)
{
/* patch the branch destination */
qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
diff --git a/tcg/tcg.c b/tcg/tcg.c
index bb890c506d..085bc73063 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -70,7 +70,7 @@
static void tcg_target_init(TCGContext *s);
static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode);
static void tcg_target_qemu_prologue(TCGContext *s);
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend);
/* The CIE and FDE header definitions will be common to all hosts. */
@@ -203,13 +203,15 @@ static TCGRegSet tcg_target_call_clobber_regs;
#if TCG_TARGET_INSN_UNIT_SIZE == 1
static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
{
- *s->code_ptr++ = v;
+ *tcg_code_ptr_rw(s, s->code_ptr) = v;
+ s->code_ptr++;
}
-static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
+static __attribute__((unused)) inline void tcg_patch8(TCGContext *s,
+ tcg_insn_unit *p,
uint8_t v)
{
- *p = v;
+ *tcg_code_ptr_rw(s, p) = v;
}
#endif
@@ -217,21 +219,23 @@ static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
{
if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
- *s->code_ptr++ = v;
+ *tcg_code_ptr_rw(s, s->code_ptr) = v;
+ s->code_ptr++;
} else {
tcg_insn_unit *p = s->code_ptr;
- memcpy(p, &v, sizeof(v));
+ memcpy(tcg_code_ptr_rw(s, p), &v, sizeof(v));
s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
}
}
-static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
+static __attribute__((unused)) inline void tcg_patch16(TCGContext *s,
+ tcg_insn_unit *p,
uint16_t v)
{
if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
- *p = v;
+ *tcg_code_ptr_rw(s, p) = v;
} else {
- memcpy(p, &v, sizeof(v));
+ memcpy(tcg_code_ptr_rw(s, p), &v, sizeof(v));
}
}
#endif
@@ -240,21 +244,23 @@ static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
{
if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
- *s->code_ptr++ = v;
+ *tcg_code_ptr_rw(s, s->code_ptr) = v;
+ s->code_ptr++;
} else {
tcg_insn_unit *p = s->code_ptr;
- memcpy(p, &v, sizeof(v));
+ memcpy(tcg_code_ptr_rw(s, p), &v, sizeof(v));
s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
}
}
-static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
+static __attribute__((unused)) inline void tcg_patch32(TCGContext *s,
+ tcg_insn_unit *p,
uint32_t v)
{
if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
*p = v;
} else {
- memcpy(p, &v, sizeof(v));
+ memcpy(tcg_code_ptr_rw(s, p), &v, sizeof(v));
}
}
#endif
@@ -263,21 +269,23 @@ static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
{
if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
- *s->code_ptr++ = v;
+ *tcg_code_ptr_rw(s, s->code_ptr) = v;
+ s->code_ptr++;
} else {
tcg_insn_unit *p = s->code_ptr;
- memcpy(p, &v, sizeof(v));
+ memcpy(tcg_code_ptr_rw(s, p), &v, sizeof(v));
s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
}
}
-static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
+static __attribute__((unused)) inline void tcg_patch64(TCGContext *s,
+ tcg_insn_unit *p,
uint64_t v)
{
if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
*p = v;
} else {
- memcpy(p, &v, sizeof(v));
+ memcpy(tcg_code_ptr_rw(s, p), &v, sizeof(v));
}
}
#endif
@@ -325,7 +333,7 @@ static bool tcg_resolve_relocs(TCGContext *s)
uintptr_t value = l->u.value;
QSIMPLEQ_FOREACH(r, &l->relocs, next) {
- if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
+ if (!patch_reloc(s, r->ptr, r->type, value, r->addend)) {
return false;
}
}
@@ -1039,7 +1047,7 @@ TranslationBlock *tcg_tb_alloc(TCGContext *s)
}
qatomic_set(&s->code_gen_ptr, next);
s->data_gen_ptr = NULL;
- return tb;
+ return (TranslationBlock *)tcg_mirror_ptr_rw(s, tb);
}
void tcg_prologue_init(TCGContext *s)
@@ -1076,6 +1084,10 @@ void tcg_prologue_init(TCGContext *s)
#endif
buf1 = s->code_ptr;
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range((uintptr_t)tcg_mirror_ptr_rw(s, buf0),
+ (uintptr_t)tcg_mirror_ptr_rw(s, buf1));
+#endif
flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
/* Deduct the prologue from the buffer. */
@@ -4267,6 +4279,12 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
return -2;
}
+#if defined(CONFIG_MIRROR_JIT)
+ /* flush data cache on mirror */
+ flush_dcache_range((uintptr_t)tcg_mirror_ptr_rw(s, s->code_buf),
+ (uintptr_t)tcg_mirror_ptr_rw(s, s->code_ptr));
+#endif
+
/* flush instruction cache */
flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc
index 1aa5f37fc6..2956c19ca5 100644
--- a/tcg/aarch64/tcg-target.c.inc
+++ b/tcg/aarch64/tcg-target.c.inc
@@ -78,40 +78,44 @@ static const int tcg_target_call_oarg_regs[1] = {
#define TCG_REG_GUEST_BASE TCG_REG_X28
#endif
-static inline bool reloc_pc26(tcg_insn_unit *code_ptr,
+static inline bool reloc_pc26(TCGContext *s,
+ tcg_insn_unit *code_ptr,
const tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
if (offset == sextract64(offset, 0, 26)) {
/* read instruction, mask away previous PC_REL26 parameter contents,
set the proper offset, then write back the instruction. */
- *code_ptr = deposit32(*code_ptr, 0, 26, offset);
+ *tcg_code_ptr_rw(s, code_ptr) =
+ deposit32(*tcg_code_ptr_rw(s, code_ptr), 0, 26, offset);
return true;
}
return false;
}
-static inline bool reloc_pc19(tcg_insn_unit *code_ptr,
+static inline bool reloc_pc19(TCGContext *s,
+ tcg_insn_unit *code_ptr,
const tcg_insn_unit *target)
{
ptrdiff_t offset = target - code_ptr;
if (offset == sextract64(offset, 0, 19)) {
- *code_ptr = deposit32(*code_ptr, 5, 19, offset);
+ *tcg_code_ptr_rw(s, code_ptr) =
+ deposit32(*tcg_code_ptr_rw(s, code_ptr), 5, 19, offset);
return true;
}
return false;
}
-static inline bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static inline bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
tcg_debug_assert(addend == 0);
switch (type) {
case R_AARCH64_JUMP26:
case R_AARCH64_CALL26:
- return reloc_pc26(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc26(s, code_ptr, (tcg_insn_unit *)value);
case R_AARCH64_CONDBR19:
- return reloc_pc19(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc19(s, code_ptr, (tcg_insn_unit *)value);
default:
g_assert_not_reached();
}
@@ -1343,7 +1347,7 @@ static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *target)
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+ uintptr_t addr, uintptr_t wr_addr)
{
tcg_insn_unit i1, i2;
TCGType rt = TCG_TYPE_I64;
@@ -1364,7 +1368,10 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
}
pair = (uint64_t)i2 << 32 | i1;
- qatomic_set((uint64_t *)jmp_addr, pair);
+ qatomic_set((uint64_t *)wr_addr, pair);
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 8);
+#endif
flush_icache_range(jmp_addr, jmp_addr + 8);
}
@@ -1583,7 +1590,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
MemOp size = opc & MO_SIZE;
- if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc19(s, lb->label_ptr[0], s->code_ptr)) {
return false;
}
@@ -1608,7 +1615,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
MemOp size = opc & MO_SIZE;
- if (!reloc_pc19(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc19(s, lb->label_ptr[0], s->code_ptr)) {
return false;
}
@@ -2919,11 +2926,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out_insn(s, 3207, RET, TCG_REG_LR);
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
int i;
for (i = 0; i < count; ++i) {
- p[i] = NOP;
+ (tcg_code_ptr_rw(s, p))[i] = NOP;
}
}
diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc
index 5ceb06a971..ee8ab72c2e 100644
--- a/tcg/arm/tcg-target.c.inc
+++ b/tcg/arm/tcg-target.c.inc
@@ -187,18 +187,21 @@ static const uint8_t tcg_cond_to_arm_cond[] = {
[TCG_COND_GTU] = COND_HI,
};
-static inline bool reloc_pc24(tcg_insn_unit *code_ptr,
+static inline bool reloc_pc24(TCGContext *s,
+ tcg_insn_unit *code_ptr,
const tcg_insn_unit *target)
{
ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
if (offset == sextract32(offset, 0, 24)) {
- *code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff);
+ *tcg_code_ptr_rw(s, code_ptr) =
+ (*tcg_code_ptr_rw(s, code_ptr) & ~0xffffff) | (offset & 0xffffff);
return true;
}
return false;
}
-static inline bool reloc_pc13(tcg_insn_unit *code_ptr,
+static inline bool reloc_pc13(TCGContext *s,
+ tcg_insn_unit *code_ptr,
const tcg_insn_unit *target)
{
ptrdiff_t offset = tcg_ptr_byte_diff(target, code_ptr) - 8;
@@ -211,21 +214,21 @@ static inline bool reloc_pc13(tcg_insn_unit *code_ptr,
}
insn = deposit32(insn, 23, 1, u);
insn = deposit32(insn, 0, 12, offset);
- *code_ptr = insn;
+ *tcg_code_ptr_rw(s, code_ptr) = insn;
return true;
}
return false;
}
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
tcg_debug_assert(addend == 0);
if (type == R_ARM_PC24) {
- return reloc_pc24(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc24(s, code_ptr, (tcg_insn_unit *)value);
} else if (type == R_ARM_PC13) {
- return reloc_pc13(code_ptr, (tcg_insn_unit *)value);
+ return reloc_pc13(s, code_ptr, (tcg_insn_unit *)value);
} else {
g_assert_not_reached();
}
@@ -1350,7 +1353,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
void *func;
- if (!reloc_pc24(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc24(s, lb->label_ptr[0], s->code_ptr)) {
return false;
}
@@ -1413,7 +1416,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
TCGMemOpIdx oi = lb->oi;
MemOp opc = get_memop(oi);
- if (!reloc_pc24(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc24(s, lb->label_ptr[0], s->code_ptr)) {
return false;
}
@@ -2257,11 +2260,11 @@ static inline void tcg_out_movi(TCGContext *s, TCGType type,
tcg_out_movi32(s, COND_AL, ret, arg);
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
int i;
for (i = 0; i < count; ++i) {
- p[i] = INSN_NOP;
+ (tcg_code_ptr_rw(s, p))[i] = INSN_NOP;
}
}
diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc
index 69acb51f5d..76bdaf748f 100644
--- a/tcg/i386/tcg-target.c.inc
+++ b/tcg/i386/tcg-target.c.inc
@@ -167,7 +167,7 @@ static bool have_lzcnt;
static const tcg_insn_unit *tb_ret_addr;
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
value += addend;
@@ -179,14 +179,14 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
}
/* FALLTHRU */
case R_386_32:
- tcg_patch32(code_ptr, value);
+ tcg_patch32(s, code_ptr, value);
break;
case R_386_PC8:
value -= (uintptr_t)code_ptr;
if (value != (int8_t)value) {
return false;
}
- tcg_patch8(code_ptr, value);
+ tcg_patch8(s, code_ptr, value);
break;
default:
tcg_abort();
@@ -1805,9 +1805,9 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
int rexw = (l->type == TCG_TYPE_I64 ? P_REXW : 0);
/* resolve label address */
- tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
+ tcg_patch32(s, label_ptr[0], s->code_ptr - label_ptr[0] - 4);
if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
- tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
+ tcg_patch32(s, label_ptr[1], s->code_ptr - label_ptr[1] - 4);
}
if (TCG_TARGET_REG_BITS == 32) {
@@ -1890,9 +1890,9 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
TCGReg retaddr;
/* resolve label address */
- tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
+ tcg_patch32(s, label_ptr[0], s->code_ptr - label_ptr[0] - 4);
if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
- tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
+ tcg_patch32(s, label_ptr[1], s->code_ptr - label_ptr[1] - 4);
}
if (TCG_TARGET_REG_BITS == 32) {
@@ -3842,9 +3842,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out_opc(s, OPC_RET, 0, 0, 0);
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
- memset(p, 0x90, count);
+ memset(tcg_code_ptr_rw(s, p), 0x90, count);
}
static void tcg_target_init(TCGContext *s)
diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc
index ace413afde..b86f81d403 100644
--- a/tcg/mips/tcg-target.c.inc
+++ b/tcg/mips/tcg-target.c.inc
@@ -153,10 +153,12 @@ static inline uint32_t reloc_pc16_val(const tcg_insn_unit *pc,
return disp & 0xffff;
}
-static inline void reloc_pc16(tcg_insn_unit *pc,
+static inline void reloc_pc16(TCGContext *s,
+ tcg_insn_unit *pc,
const tcg_insn_unit *target)
{
- *pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target));
+ *tcg_code_ptr_rw(s, pc) =
+ deposit32(*tcg_code_ptr_rw(s, pc), 0, 16, reloc_pc16_val(pc, target));
}
static inline uint32_t reloc_26_val(const tcg_insn_unit *pc,
@@ -166,18 +168,20 @@ static inline uint32_t reloc_26_val(const tcg_insn_unit *pc,
return ((uintptr_t)target >> 2) & 0x3ffffff;
}
-static inline void reloc_26(tcg_insn_unit *pc,
+static inline void reloc_26(TCGContext *s,
+ tcg_insn_unit *pc,
const tcg_insn_unit *target)
{
- *pc = deposit32(*pc, 0, 26, reloc_26_val(pc, target));
+ *tcg_code_ptr_rw(s, pc) =
+ deposit32(*tcg_code_ptr_rw(s, pc), 0, 26, reloc_26_val(pc, target));
}
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
tcg_debug_assert(type == R_MIPS_PC16);
tcg_debug_assert(addend == 0);
- reloc_pc16(code_ptr, (tcg_insn_unit *)value);
+ reloc_pc16(s, code_ptr, (tcg_insn_unit *)value);
return true;
}
@@ -929,7 +933,7 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
tcg_out_opc_br(s, b_opc, arg1, arg2);
if (l->has_value) {
- reloc_pc16(s->code_ptr - 1, l->u.value_ptr);
+ reloc_pc16(s, s->code_ptr - 1, l->u.value_ptr);
} else {
tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, l, 0);
}
@@ -1320,9 +1324,9 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
int i;
/* resolve label address */
- reloc_pc16(l->label_ptr[0], s->code_ptr);
+ reloc_pc16(s, l->label_ptr[0], s->code_ptr);
if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
- reloc_pc16(l->label_ptr[1], s->code_ptr);
+ reloc_pc16(s, l->label_ptr[1], s->code_ptr);
}
i = 1;
@@ -1350,7 +1354,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
}
tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
- reloc_pc16(s->code_ptr - 1, l->raddr);
+ reloc_pc16(s, s->code_ptr - 1, l->raddr);
/* delay slot */
if (TCG_TARGET_REG_BITS == 64 && l->type == TCG_TYPE_I32) {
@@ -1370,9 +1374,9 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
int i;
/* resolve label address */
- reloc_pc16(l->label_ptr[0], s->code_ptr);
+ reloc_pc16(s, l->label_ptr[0], s->code_ptr);
if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
- reloc_pc16(l->label_ptr[1], s->code_ptr);
+ reloc_pc16(s, l->label_ptr[1], s->code_ptr);
}
i = 1;
@@ -2662,9 +2666,12 @@ static void tcg_target_init(TCGContext *s)
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+ uintptr_t addr, uintptr_t wr_addr)
{
- qatomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
+ qatomic_set((uint32_t *)wr_addr, deposit32(OPC_J, 0, 26, addr >> 2));
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 4);
+#endif
flush_icache_range(jmp_addr, jmp_addr + 4);
}
diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc
index 945ff2fba1..262def4428 100644
--- a/tcg/ppc/tcg-target.c.inc
+++ b/tcg/ppc/tcg-target.c.inc
@@ -192,12 +192,14 @@ static uint32_t reloc_pc24_val(const tcg_insn_unit *pc,
return disp & 0x3fffffc;
}
-static bool reloc_pc24(tcg_insn_unit *pc,
+static bool reloc_pc24(TCGContext *s,
+ tcg_insn_unit *pc,
const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
if (in_range_b(disp)) {
- *pc = (*pc & ~0x3fffffc) | (disp & 0x3fffffc);
+ *tcg_code_ptr_rw(s, pc) =
+ (*tcg_code_ptr_rw(s, pc) & ~0x3fffffc) | (disp & 0x3fffffc);
return true;
}
return false;
@@ -211,12 +213,14 @@ static uint16_t reloc_pc14_val(const tcg_insn_unit *pc,
return disp & 0xfffc;
}
-static bool reloc_pc14(tcg_insn_unit *pc,
+static bool reloc_pc14(TCGContext *s,
+ tcg_insn_unit *pc,
const tcg_insn_unit *target)
{
ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
if (disp == (int16_t) disp) {
- *pc = (*pc & ~0xfffc) | (disp & 0xfffc);
+ *tcg_code_ptr_rw(s, pc) =
+ (*tcg_code_ptr_rw(s, pc) & ~0xfffc) | (disp & 0xfffc);
return true;
}
return false;
@@ -674,7 +678,7 @@ static const uint32_t tcg_to_isel[] = {
[TCG_COND_GTU] = ISEL | BC_(7, CR_GT),
};
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
tcg_insn_unit *target;
@@ -686,9 +690,9 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
switch (type) {
case R_PPC_REL14:
- return reloc_pc14(code_ptr, target);
+ return reloc_pc14(s, code_ptr, target);
case R_PPC_REL24:
- return reloc_pc24(code_ptr, target);
+ return reloc_pc24(s, code_ptr, target);
case R_PPC_ADDR16:
/*
* We are (slightly) abusing this relocation type. In particular,
@@ -1727,7 +1731,7 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0)
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+ uintptr_t addr, uintptr_t wr_addr)
{
if (TCG_TARGET_REG_BITS == 64) {
tcg_insn_unit i1, i2;
@@ -1756,12 +1760,18 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
/* As per the enclosing if, this is ppc64. Avoid the _Static_assert
within qatomic_set that would fail to build a ppc32 host. */
- qatomic_set__nocheck((uint64_t *)jmp_addr, pair);
+ qatomic_set__nocheck((uint64_t *)wr_addr, pair);
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 8);
+#endif
flush_icache_range(jmp_addr, jmp_addr + 8);
} else {
intptr_t diff = addr - jmp_addr;
tcg_debug_assert(in_range_b(diff));
- qatomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
+ qatomic_set((uint32_t *)wr_addr, B | (diff & 0x3fffffc));
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 8);
+#endif
flush_icache_range(jmp_addr, jmp_addr + 4);
}
}
@@ -2012,7 +2022,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp opc = get_memop(oi);
TCGReg hi, lo, arg = TCG_REG_R3;
- if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc14(s, lb->label_ptr[0], s->code_ptr)) {
return false;
}
@@ -2060,7 +2070,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
MemOp s_bits = opc & MO_SIZE;
TCGReg hi, lo, arg = TCG_REG_R3;
- if (!reloc_pc14(lb->label_ptr[0], s->code_ptr)) {
+ if (!reloc_pc14(s, lb->label_ptr[0], s->code_ptr)) {
return false;
}
@@ -2257,11 +2267,11 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
#endif
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
int i;
for (i = 0; i < count; ++i) {
- p[i] = NOP;
+ (tcg_code_ptr_rw(s, p))[i] = NOP;
}
}
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc
index 5432ee89db..19882db57f 100644
--- a/tcg/riscv/tcg-target.c.inc
+++ b/tcg/riscv/tcg-target.c.inc
@@ -413,11 +413,11 @@ static void tcg_out_opc_jump(TCGContext *s, RISCVInsn opc,
tcg_out32(s, encode_uj(opc, rd, imm));
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
int i;
for (i = 0; i < count; ++i) {
- p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0);
+ (tcg_code_ptr_rw(s, p))[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0);
}
}
@@ -425,46 +425,52 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
* Relocations
*/
-static bool reloc_sbimm12(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_sbimm12(TCGContext *s,
+ tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
intptr_t offset = (intptr_t)target - (intptr_t)code_ptr;
if (offset == sextreg(offset, 1, 12) << 1) {
- code_ptr[0] |= encode_sbimm12(offset);
+ (tcg_code_ptr_rw(s, code_ptr))[0] |= encode_sbimm12(offset);
return true;
}
return false;
}
-static bool reloc_jimm20(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_jimm20(TCGContext *s,
+ tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
intptr_t offset = (intptr_t)target - (intptr_t)code_ptr;
if (offset == sextreg(offset, 1, 20) << 1) {
- code_ptr[0] |= encode_ujimm20(offset);
+ (tcg_code_ptr_rw(s, code_ptr))[0] |= encode_ujimm20(offset);
return true;
}
return false;
}
-static bool reloc_call(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+static bool reloc_call(TCGContext *s,
+ tcg_insn_unit *code_ptr,
+ const tcg_insn_unit *target)
{
intptr_t offset = (intptr_t)target - (intptr_t)code_ptr;
int32_t lo = sextreg(offset, 0, 12);
int32_t hi = offset - lo;
if (offset == hi + lo) {
- code_ptr[0] |= encode_uimm20(hi);
- code_ptr[1] |= encode_imm12(lo);
+ (tcg_code_ptr_rw(s, code_ptr))[0] |= encode_uimm20(hi);
+ (tcg_code_ptr_rw(s, code_ptr))[1] |= encode_imm12(lo);
return true;
}
return false;
}
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
uint32_t insn = *code_ptr;
@@ -478,7 +484,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
diff = value - (uintptr_t)code_ptr;
short_jmp = diff == sextreg(diff, 0, 12);
if (short_jmp) {
- return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value);
+ return reloc_sbimm12(s, code_ptr, (tcg_insn_unit *)value);
} else {
/* Invert the condition */
insn = insn ^ (1 << 12);
@@ -499,9 +505,9 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
}
break;
case R_RISCV_JAL:
- return reloc_jimm20(code_ptr, (tcg_insn_unit *)value);
+ return reloc_jimm20(s, code_ptr, (tcg_insn_unit *)value);
case R_RISCV_CALL:
- return reloc_call(code_ptr, (tcg_insn_unit *)value);
+ return reloc_call(s, code_ptr, (tcg_insn_unit *)value);
default:
tcg_abort();
}
@@ -557,7 +563,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
if (tmp == (int32_t)tmp) {
tcg_out_opc_upper(s, OPC_AUIPC, rd, 0);
tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0);
- ret = reloc_call(s->code_ptr - 2, (tcg_insn_unit *)val);
+ ret = reloc_call(s, s->code_ptr - 2, (tcg_insn_unit *)val);
tcg_debug_assert(ret == true);
return;
}
@@ -875,7 +881,7 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail)
/* long jump: -2147483646 to 2147483648 */
tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0);
tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0);
- ret = reloc_call(s->code_ptr - 2, arg);\
+ ret = reloc_call(s, s->code_ptr - 2, arg);\
tcg_debug_assert(ret == true);
} else if (TCG_TARGET_REG_BITS == 64) {
/* far jump: 64-bit */
@@ -1053,7 +1059,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
}
/* resolve label address */
- if (!patch_reloc(l->label_ptr[0], R_RISCV_BRANCH,
+ if (!patch_reloc(s, l->label_ptr[0], R_RISCV_BRANCH,
(intptr_t) s->code_ptr, 0)) {
return false;
}
@@ -1088,7 +1094,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
}
/* resolve label address */
- if (!patch_reloc(l->label_ptr[0], R_RISCV_BRANCH,
+ if (!patch_reloc(s, l->label_ptr[0], R_RISCV_BRANCH,
(intptr_t) s->code_ptr, 0)) {
return false;
}
diff --git a/tcg/s390/tcg-target.c.inc b/tcg/s390/tcg-target.c.inc
index 7c6917f3c4..f9498ddf63 100644
--- a/tcg/s390/tcg-target.c.inc
+++ b/tcg/s390/tcg-target.c.inc
@@ -366,7 +366,7 @@ static void * const qemu_st_helpers[16] = {
static const tcg_insn_unit *tb_ret_addr;
uint64_t s390_facilities;
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
intptr_t pcrel2;
@@ -378,13 +378,13 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
switch (type) {
case R_390_PC16DBL:
if (pcrel2 == (int16_t)pcrel2) {
- tcg_patch16(code_ptr, pcrel2);
+ tcg_patch16(s, code_ptr, pcrel2);
return true;
}
break;
case R_390_PC32DBL:
if (pcrel2 == (int32_t)pcrel2) {
- tcg_patch32(code_ptr, pcrel2);
+ tcg_patch32(s, code_ptr, pcrel2);
return true;
}
break;
@@ -392,7 +392,7 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
if (value == sextract64(value, 0, 20)) {
old = *(uint32_t *)code_ptr & 0xf00000ff;
old |= ((value & 0xfff) << 16) | ((value & 0xff000) >> 4);
- tcg_patch32(code_ptr, old);
+ tcg_patch32(s, code_ptr, old);
return true;
}
break;
@@ -1613,7 +1613,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
TCGMemOpIdx oi = lb->oi;
MemOp opc = get_memop(oi);
- if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
+ if (!patch_reloc(s, lb->label_ptr[0], R_390_PC16DBL,
(intptr_t)s->code_ptr, 2)) {
return false;
}
@@ -1638,7 +1638,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
TCGMemOpIdx oi = lb->oi;
MemOp opc = get_memop(oi);
- if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
+ if (!patch_reloc(s, lb->label_ptr[0], R_390_PC16DBL,
(intptr_t)s->code_ptr, 2)) {
return false;
}
@@ -2576,9 +2576,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
- memset(p, 0x07, count * sizeof(tcg_insn_unit));
+ memset(tcg_code_ptr_rw(s, p), 0x07, count * sizeof(tcg_insn_unit));
}
typedef struct {
diff --git a/tcg/sparc/tcg-target.c.inc b/tcg/sparc/tcg-target.c.inc
index 5eac18b719..ebfc452144 100644
--- a/tcg/sparc/tcg-target.c.inc
+++ b/tcg/sparc/tcg-target.c.inc
@@ -291,14 +291,14 @@ static inline int check_fit_i32(int32_t val, unsigned int bits)
# define check_fit_ptr check_fit_i32
#endif
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
uint32_t insn = *code_ptr;
intptr_t pcrel;
value += addend;
- pcrel = tcg_ptr_byte_diff((tcg_insn_unit *)value, code_ptr);
+ pcrel = tcg_ptr_byte_diff((const tcg_insn_unit *)value, code_ptr);
switch (type) {
case R_SPARC_WDISP16:
@@ -1048,11 +1048,11 @@ static void tcg_target_qemu_prologue(TCGContext *s)
#endif
}
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count)
{
int i;
for (i = 0; i < count; ++i) {
- p[i] = NOP;
+ (tcg_code_ptr_rw(s, p))[i] = NOP;
}
}
@@ -1226,7 +1226,8 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
}
}
- *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr));
+ *tcg_code_ptr_rw(s, label_ptr) |=
+ INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr));
#else
if (SPARC64 && TARGET_LONG_BITS == 32) {
tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
@@ -1822,7 +1823,7 @@ void tcg_register_jit(void *buf, size_t buf_size)
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
- uintptr_t addr)
+ uintptr_t addr, uintptr_t wr_addr)
{
intptr_t tb_disp = addr - tc_ptr;
intptr_t br_disp = addr - jmp_addr;
@@ -1834,8 +1835,11 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
tcg_debug_assert(br_disp == (int32_t)br_disp);
if (!USE_REG_TB) {
- qatomic_set((uint32_t *)jmp_addr,
+ qatomic_set((uint32_t *)wr_addr,
deposit32(CALL, 0, 30, br_disp >> 2));
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 4);
+#endif
flush_icache_range(jmp_addr, jmp_addr + 4);
return;
}
@@ -1859,6 +1863,9 @@ void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
| INSN_IMM13((tb_disp & 0x3ff) | -0x400));
}
- qatomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1));
+ qatomic_set((uint64_t *)wr_addr, deposit64(i2, 32, 32, i1));
+#if defined(CONFIG_MIRROR_JIT)
+ flush_dcache_range(wr_addr, wr_addr + 8);
+#endif
flush_icache_range(jmp_addr, jmp_addr + 8);
}
diff --git a/tcg/tcg-pool.c.inc b/tcg/tcg-pool.c.inc
index 82cbcc89bd..8434496e72 100644
--- a/tcg/tcg-pool.c.inc
+++ b/tcg/tcg-pool.c.inc
@@ -119,7 +119,7 @@ static inline void new_pool_l8(TCGContext *s, int rtype, tcg_insn_unit *label,
}
/* To be provided by cpu/tcg-target.c.inc. */
-static void tcg_out_nop_fill(tcg_insn_unit *p, int count);
+static void tcg_out_nop_fill(TCGContext *s, tcg_insn_unit *p, int count);
static int tcg_out_pool_finalize(TCGContext *s)
{
@@ -135,7 +135,7 @@ static int tcg_out_pool_finalize(TCGContext *s)
again when allocating the next TranslationBlock structure. */
a = (void *)ROUND_UP((uintptr_t)s->code_ptr,
sizeof(tcg_target_ulong) * p->nlong);
- tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr);
+ tcg_out_nop_fill(s, s->code_ptr, (tcg_insn_unit *)a - s->code_ptr);
s->data_gen_ptr = a;
for (; p != NULL; p = p->next) {
@@ -144,11 +144,12 @@ static int tcg_out_pool_finalize(TCGContext *s)
if (unlikely(a > s->code_gen_highwater)) {
return -1;
}
- memcpy(a, p->data, size);
+ memcpy(tcg_mirror_ptr_rw(s, a), p->data, size);
a += size;
l = p;
}
- if (!patch_reloc(p->label, p->rtype, (intptr_t)a - size, p->addend)) {
+ if (!patch_reloc(s, p->label, p->rtype,
+ (intptr_t)a - size, p->addend)) {
return -2;
}
}
diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc
index d5a4d9d37c..133213be3a 100644
--- a/tcg/tci/tcg-target.c.inc
+++ b/tcg/tci/tcg-target.c.inc
@@ -369,7 +369,7 @@ static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
};
#endif
-static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
+static bool patch_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend)
{
/* tcg_out_reloc always uses the same type, addend. */
@@ -377,9 +377,9 @@ static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
tcg_debug_assert(addend == 0);
tcg_debug_assert(value != 0);
if (TCG_TARGET_REG_BITS == 32) {
- tcg_patch32(code_ptr, value);
+ tcg_patch32(s, code_ptr, value);
} else {
- tcg_patch64(code_ptr, value);
+ tcg_patch64(s, code_ptr, value);
}
return true;
}
--
2.28.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 4/4] tcg: make mirror mapped JIT optional
2020-10-26 17:25 [PATCH 0/4] Mirror map JIT memory for TCG Joelle van Dyne
` (2 preceding siblings ...)
2020-10-26 17:25 ` [PATCH 3/4] tcg: use mirror map JIT in code generation Joelle van Dyne
@ 2020-10-26 17:25 ` Joelle van Dyne
3 siblings, 0 replies; 5+ messages in thread
From: Joelle van Dyne @ 2020-10-26 17:25 UTC (permalink / raw)
To: qemu-devel
Cc: Paolo Bonzini, Laurent Vivier, Joelle van Dyne, Richard Henderson
On platforms that support mirror mapping, give the user the option to
disable it. There is a performance penalty for mirror JIT and on some
platforms, system security must be downgraded to use QEMU with mirror JIT
turned off.
To aid in testing, we still enable mirror JIT by default when configured
with --enable-debug-tcg.
Signed-off-by: Joelle van Dyne <j@getutm.app>
---
include/sysemu/tcg.h | 2 +-
accel/tcg/tcg-all.c | 31 ++++++++++++++++++++-
accel/tcg/translate-all.c | 57 ++++++++++++++++++++++++++-------------
bsd-user/main.c | 2 +-
linux-user/main.c | 2 +-
qemu-options.hx | 11 ++++++++
6 files changed, 82 insertions(+), 23 deletions(-)
diff --git a/include/sysemu/tcg.h b/include/sysemu/tcg.h
index d9d3ca8559..569f90b11d 100644
--- a/include/sysemu/tcg.h
+++ b/include/sysemu/tcg.h
@@ -8,7 +8,7 @@
#ifndef SYSEMU_TCG_H
#define SYSEMU_TCG_H
-void tcg_exec_init(unsigned long tb_size);
+void tcg_exec_init(unsigned long tb_size, bool mirror_rwx);
#ifdef CONFIG_TCG
extern bool tcg_allowed;
#define tcg_enabled() (tcg_allowed)
diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c
index fa1208158f..52fccd9097 100644
--- a/accel/tcg/tcg-all.c
+++ b/accel/tcg/tcg-all.c
@@ -39,6 +39,7 @@ struct TCGState {
bool mttcg_enabled;
unsigned long tb_size;
+ bool mirror_jit;
};
typedef struct TCGState TCGState;
@@ -94,6 +95,11 @@ static void tcg_accel_instance_init(Object *obj)
TCGState *s = TCG_STATE(obj);
s->mttcg_enabled = default_mttcg_enabled();
+#if defined(CONFIG_MIRROR_JIT) && defined(CONFIG_DEBUG_TCG)
+ s->mirror_jit = true;
+#else
+ s->mirror_jit = false;
+#endif
}
bool mttcg_enabled;
@@ -102,7 +108,7 @@ static int tcg_init(MachineState *ms)
{
TCGState *s = TCG_STATE(current_accel());
- tcg_exec_init(s->tb_size * 1024 * 1024);
+ tcg_exec_init(s->tb_size * 1024 * 1024, s->mirror_jit);
mttcg_enabled = s->mttcg_enabled;
cpus_register_accel(&tcg_cpus);
@@ -168,6 +174,22 @@ static void tcg_set_tb_size(Object *obj, Visitor *v,
s->tb_size = value;
}
+#ifdef CONFIG_MIRROR_JIT
+static bool tcg_get_mirror_jit(Object *obj, Error **errp)
+{
+ TCGState *s = TCG_STATE(obj);
+
+ return s->mirror_jit;
+}
+
+static void tcg_set_mirror_jit(Object *obj, bool value, Error **errp)
+{
+ TCGState *s = TCG_STATE(obj);
+
+ s->mirror_jit = value;
+}
+#endif
+
static void tcg_accel_class_init(ObjectClass *oc, void *data)
{
AccelClass *ac = ACCEL_CLASS(oc);
@@ -185,6 +207,13 @@ static void tcg_accel_class_init(ObjectClass *oc, void *data)
object_class_property_set_description(oc, "tb-size",
"TCG translation block cache size");
+#ifdef CONFIG_MIRROR_JIT
+ object_class_property_add_bool(oc, "mirror-jit",
+ tcg_get_mirror_jit, tcg_set_mirror_jit);
+ object_class_property_set_description(oc, "mirror-jit",
+ "JIT pages mapped into separate RW and RX regions");
+#endif
+
}
static const TypeInfo tcg_accel_type = {
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index bfb856ac1a..2144433024 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -1027,12 +1027,15 @@ static inline void *split_cross_256mb(void *buf1, size_t size1)
static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
__attribute__((aligned(CODE_GEN_ALIGN)));
-static inline void *alloc_code_gen_buffer(void)
+static inline void *alloc_code_gen_buffer(bool mirror_jit)
{
void *buf = static_code_gen_buffer;
void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer);
size_t size;
+ /* not applicable */
+ assert(!mirror_jit);
+
/* page-align the beginning and end of the buffer */
buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size);
end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size);
@@ -1061,14 +1064,15 @@ static inline void *alloc_code_gen_buffer(void)
return buf;
}
#elif defined(_WIN32)
-static inline void *alloc_code_gen_buffer(void)
+static inline void *alloc_code_gen_buffer(bool mirror_jit)
{
size_t size = tcg_ctx->code_gen_buffer_size;
+ assert(!mirror_jit); /* not applicable */
return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
PAGE_EXECUTE_READWRITE);
}
#else
-static inline void *alloc_code_gen_buffer(void)
+static inline void *alloc_code_gen_buffer(bool mirror_jit)
{
int prot = PROT_READ | PROT_EXEC;
int flags = 0;
@@ -1078,16 +1082,22 @@ static inline void *alloc_code_gen_buffer(void)
#if defined(CONFIG_MIRROR_JIT)
#if defined(CONFIG_LINUX)
- fd = qemu_memfd_create("tcg-jit", size, false, 0, 0, NULL);
- if (fd < 0) {
- return NULL;
+ if (mirror_jit) {
+ fd = qemu_memfd_create("tcg-jit", size, false, 0, 0, NULL);
+ if (fd < 0) {
+ return NULL;
+ }
+ tcg_ctx->code_gen_buffer_fd = fd;
+ flags |= MAP_SHARED;
+ } else {
+ prot |= PROT_WRITE;
+ flags |= MAP_ANONYMOUS | MAP_PRIVATE;
}
- tcg_ctx->code_gen_buffer_fd = fd;
- flags |= MAP_SHARED;
#else /* defined(CONFIG_LINUX) */
#error "Mirror JIT unimplemented for this platform."
#endif /* defined(CONFIG_LINUX) */
#else /* defined(CONFIG_MIRROR_JIT) */
+ assert(!mirror_jit);
prot |= PROT_WRITE;
flags |= MAP_ANONYMOUS | MAP_PRIVATE;
#endif /* defined(CONFIG_MIRROR_JIT) */
@@ -1162,7 +1172,7 @@ static inline void *alloc_jit_rw_mirror(void)
static inline void code_gen_alloc(size_t tb_size, bool mirror_jit)
{
tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size);
- tcg_ctx->code_gen_buffer = alloc_code_gen_buffer();
+ tcg_ctx->code_gen_buffer = alloc_code_gen_buffer(mirror_jit);
if (tcg_ctx->code_gen_buffer == NULL) {
fprintf(stderr, "Could not allocate dynamic translator buffer\n");
exit(1);
@@ -1170,13 +1180,20 @@ static inline void code_gen_alloc(size_t tb_size, bool mirror_jit)
#if defined(CONFIG_MIRROR_JIT)
void *mirror;
- /* For platforms that need a mirror mapping for code execution */
- mirror = alloc_jit_rw_mirror();
- if (mirror == NULL) {
- fprintf(stderr, "Could not remap code buffer mirror\n");
- exit(1);
+ if (mirror_jit) {
+ /* For platforms that need a mirror mapping for code execution */
+ mirror = alloc_jit_rw_mirror();
+ if (mirror == NULL) {
+ fprintf(stderr, "Could not remap code buffer mirror\n");
+ exit(1);
+ }
+ } else {
+ /* If we disable mirror mapping run option */
+ mirror = tcg_ctx->code_gen_buffer;
}
tcg_ctx->code_rw_mirror_diff = mirror - tcg_ctx->code_gen_buffer;
+#else
+ assert(!mirror_jit);
#endif /* CONFIG_MIRROR_JIT */
}
@@ -1201,16 +1218,18 @@ static void tb_htable_init(void)
qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
}
-/* Must be called before using the QEMU cpus. 'tb_size' is the size
- (in bytes) allocated to the translation buffer. Zero means default
- size. */
-void tcg_exec_init(unsigned long tb_size)
+/*
+ * Must be called before using the QEMU cpus. 'tb_size' is the size
+ * (in bytes) allocated to the translation buffer. Zero means default
+ * size. mirror_jit separates RX and RW allocations.
+ */
+void tcg_exec_init(unsigned long tb_size, bool mirror_jit)
{
tcg_allowed = true;
cpu_gen_init();
page_init();
tb_htable_init();
- code_gen_alloc(tb_size);
+ code_gen_alloc(tb_size, mirror_jit);
#if defined(CONFIG_SOFTMMU)
/* There's no guest base to take into account, so go ahead and
initialize the prologue now. */
diff --git a/bsd-user/main.c b/bsd-user/main.c
index ac40d79bfa..ffd4888a26 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -910,7 +910,7 @@ int main(int argc, char **argv)
}
/* init tcg before creating CPUs and to get qemu_host_page_size */
- tcg_exec_init(0);
+ tcg_exec_init(0, false);
cpu_type = parse_cpu_option(cpu_model);
cpu = cpu_create(cpu_type);
diff --git a/linux-user/main.c b/linux-user/main.c
index 75c9785157..3856b2611d 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -705,7 +705,7 @@ int main(int argc, char **argv, char **envp)
cpu_type = parse_cpu_option(cpu_model);
/* init tcg before creating CPUs and to get qemu_host_page_size */
- tcg_exec_init(0);
+ tcg_exec_init(0, false);
cpu = cpu_create(cpu_type);
env = cpu->env_ptr;
diff --git a/qemu-options.hx b/qemu-options.hx
index 1da52a269c..cd852f8df7 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -123,6 +123,9 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel,
" igd-passthru=on|off (enable Xen integrated Intel graphics passthrough, default=off)\n"
" kernel-irqchip=on|off|split controls accelerated irqchip support (default=on)\n"
" kvm-shadow-mem=size of KVM shadow MMU in bytes\n"
+#ifdef CONFIG_MIRROR_JIT
+ " mirror-jit=on|off (JIT pages mapped into separate RW and RX regions, default=off)\n"
+#endif
" tb-size=n (TCG translation block cache size)\n"
" thread=single|multi (enable multi-threaded TCG)\n", QEMU_ARCH_ALL)
SRST
@@ -148,6 +151,14 @@ SRST
``kvm-shadow-mem=size``
Defines the size of the KVM shadow MMU.
+#ifdef CONFIG_MIRROR_JIT
+ ``mirror-jit=on|off``
+ Useful for debugging TCG or running on a strict W^X platform. When
+ enabled, TB code is written to a mirror mapped RW address separate from
+ the RX address that is executed. (default=off, but on if built with
+ TCG debugging)
+
+#endif
``tb-size=n``
Controls the size (in MiB) of the TCG translation block cache.
--
2.28.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2020-10-26 18:02 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-10-26 17:25 [PATCH 0/4] Mirror map JIT memory for TCG Joelle van Dyne
2020-10-26 17:25 ` [PATCH 1/4] tcg: add const hints for code pointers Joelle van Dyne
2020-10-26 17:25 ` [PATCH 2/4] tcg: implement mirror mapped JIT for Linux Joelle van Dyne
2020-10-26 17:25 ` [PATCH 3/4] tcg: use mirror map JIT in code generation Joelle van Dyne
2020-10-26 17:25 ` [PATCH 4/4] tcg: make mirror mapped JIT optional Joelle van Dyne
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).