* [PULL 01/15] memory: Log access direction for invalid accesses
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 02/15] target/arm: Use MO_128 for 16 byte atomics Richard Henderson
` (14 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: David Hildenbrand
From: BALATON Zoltan <balaton@eik.bme.hu>
In memory_region_access_valid() invalid accesses are logged to help
debugging but the log message does not say if it was a read or write.
Log that too to better identify the access causing the problem.
Reviewed-by: David Hildenbrand <david@redhat.com>
Signed-off-by: BALATON Zoltan <balaton@eik.bme.hu>
Message-Id: <20211011173616.F1DE0756022@zero.eik.bme.hu>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
softmmu/memory.c | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/softmmu/memory.c b/softmmu/memory.c
index db182e5d3d..e5826faa0c 100644
--- a/softmmu/memory.c
+++ b/softmmu/memory.c
@@ -1378,17 +1378,17 @@ bool memory_region_access_valid(MemoryRegion *mr,
{
if (mr->ops->valid.accepts
&& !mr->ops->valid.accepts(mr->opaque, addr, size, is_write, attrs)) {
- qemu_log_mask(LOG_GUEST_ERROR, "Invalid access at addr "
- "0x%" HWADDR_PRIX ", size %u, "
- "region '%s', reason: rejected\n",
+ qemu_log_mask(LOG_GUEST_ERROR, "Invalid %s at addr 0x%" HWADDR_PRIX
+ ", size %u, region '%s', reason: rejected\n",
+ is_write ? "write" : "read",
addr, size, memory_region_name(mr));
return false;
}
if (!mr->ops->valid.unaligned && (addr & (size - 1))) {
- qemu_log_mask(LOG_GUEST_ERROR, "Invalid access at addr "
- "0x%" HWADDR_PRIX ", size %u, "
- "region '%s', reason: unaligned\n",
+ qemu_log_mask(LOG_GUEST_ERROR, "Invalid %s at addr 0x%" HWADDR_PRIX
+ ", size %u, region '%s', reason: unaligned\n",
+ is_write ? "write" : "read",
addr, size, memory_region_name(mr));
return false;
}
@@ -1400,10 +1400,10 @@ bool memory_region_access_valid(MemoryRegion *mr,
if (size > mr->ops->valid.max_access_size
|| size < mr->ops->valid.min_access_size) {
- qemu_log_mask(LOG_GUEST_ERROR, "Invalid access at addr "
- "0x%" HWADDR_PRIX ", size %u, "
- "region '%s', reason: invalid size "
- "(min:%u max:%u)\n",
+ qemu_log_mask(LOG_GUEST_ERROR, "Invalid %s at addr 0x%" HWADDR_PRIX
+ ", size %u, region '%s', reason: invalid size "
+ "(min:%u max:%u)\n",
+ is_write ? "write" : "read",
addr, size, memory_region_name(mr),
mr->ops->valid.min_access_size,
mr->ops->valid.max_access_size);
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 02/15] target/arm: Use MO_128 for 16 byte atomics
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
2021-10-13 18:22 ` [PULL 01/15] memory: Log access direction for invalid accesses Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 03/15] target/i386: " Richard Henderson
` (13 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: qemu-arm, Philippe Mathieu-Daudé
Cc: qemu-arm@nongnu.org
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-a64.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
index c5af779006..4cafd3c11a 100644
--- a/target/arm/helper-a64.c
+++ b/target/arm/helper-a64.c
@@ -560,7 +560,7 @@ uint64_t HELPER(paired_cmpxchg64_le_parallel)(CPUARMState *env, uint64_t addr,
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+ oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
cmpv = int128_make128(env->exclusive_val, env->exclusive_high);
newv = int128_make128(new_lo, new_hi);
@@ -630,7 +630,7 @@ uint64_t HELPER(paired_cmpxchg64_be_parallel)(CPUARMState *env, uint64_t addr,
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
+ oi = make_memop_idx(MO_BE | MO_128 | MO_ALIGN, mem_idx);
/*
* High and low need to be switched here because this is not actually a
@@ -656,7 +656,7 @@ void HELPER(casp_le_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+ oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
cmpv = int128_make128(env->xregs[rs], env->xregs[rs + 1]);
newv = int128_make128(new_lo, new_hi);
@@ -677,7 +677,7 @@ void HELPER(casp_be_parallel)(CPUARMState *env, uint32_t rs, uint64_t addr,
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
+ oi = make_memop_idx(MO_LE | MO_128 | MO_ALIGN, mem_idx);
cmpv = int128_make128(env->xregs[rs + 1], env->xregs[rs]);
newv = int128_make128(new_lo, new_hi);
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 03/15] target/i386: Use MO_128 for 16 byte atomics
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
2021-10-13 18:22 ` [PULL 01/15] memory: Log access direction for invalid accesses Richard Henderson
2021-10-13 18:22 ` [PULL 02/15] target/arm: Use MO_128 for 16 byte atomics Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 04/15] target/ppc: " Richard Henderson
` (12 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/i386/tcg/mem_helper.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/target/i386/tcg/mem_helper.c b/target/i386/tcg/mem_helper.c
index 0fd696f9c1..a207e624cb 100644
--- a/target/i386/tcg/mem_helper.c
+++ b/target/i386/tcg/mem_helper.c
@@ -136,7 +136,7 @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
Int128 newv = int128_make128(env->regs[R_EBX], env->regs[R_ECX]);
int mem_idx = cpu_mmu_index(env, false);
- MemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+ MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
Int128 oldv = cpu_atomic_cmpxchgo_le_mmu(env, a0, cmpv, newv, oi, ra);
if (int128_eq(oldv, cmpv)) {
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 04/15] target/ppc: Use MO_128 for 16 byte atomics
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (2 preceding siblings ...)
2021-10-13 18:22 ` [PULL 03/15] target/i386: " Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 05/15] target/s390x: " Richard Henderson
` (11 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: qemu-ppc, Philippe Mathieu-Daudé
Cc: qemu-ppc@nongnu.org
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/ppc/translate.c | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/target/ppc/translate.c b/target/ppc/translate.c
index b985e9e55b..9ca78ee156 100644
--- a/target/ppc/translate.c
+++ b/target/ppc/translate.c
@@ -3462,10 +3462,12 @@ static void gen_std(DisasContext *ctx)
if (HAVE_ATOMIC128) {
TCGv_i32 oi = tcg_temp_new_i32();
if (ctx->le_mode) {
- tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ, ctx->mem_idx));
+ tcg_gen_movi_i32(oi, make_memop_idx(MO_LE | MO_128,
+ ctx->mem_idx));
gen_helper_stq_le_parallel(cpu_env, EA, lo, hi, oi);
} else {
- tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ, ctx->mem_idx));
+ tcg_gen_movi_i32(oi, make_memop_idx(MO_BE | MO_128,
+ ctx->mem_idx));
gen_helper_stq_be_parallel(cpu_env, EA, lo, hi, oi);
}
tcg_temp_free_i32(oi);
@@ -4067,11 +4069,11 @@ static void gen_lqarx(DisasContext *ctx)
if (HAVE_ATOMIC128) {
TCGv_i32 oi = tcg_temp_new_i32();
if (ctx->le_mode) {
- tcg_gen_movi_i32(oi, make_memop_idx(MO_LEQ | MO_ALIGN_16,
+ tcg_gen_movi_i32(oi, make_memop_idx(MO_LE | MO_128 | MO_ALIGN,
ctx->mem_idx));
gen_helper_lq_le_parallel(lo, cpu_env, EA, oi);
} else {
- tcg_gen_movi_i32(oi, make_memop_idx(MO_BEQ | MO_ALIGN_16,
+ tcg_gen_movi_i32(oi, make_memop_idx(MO_BE | MO_128 | MO_ALIGN,
ctx->mem_idx));
gen_helper_lq_be_parallel(lo, cpu_env, EA, oi);
}
@@ -4122,7 +4124,7 @@ static void gen_stqcx_(DisasContext *ctx)
if (tb_cflags(ctx->base.tb) & CF_PARALLEL) {
if (HAVE_CMPXCHG128) {
- TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_Q) | MO_ALIGN_16);
+ TCGv_i32 oi = tcg_const_i32(DEF_MEMOP(MO_128) | MO_ALIGN);
if (ctx->le_mode) {
gen_helper_stqcx_le_parallel(cpu_crf[0], cpu_env,
EA, lo, hi, oi);
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 05/15] target/s390x: Use MO_128 for 16 byte atomics
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (3 preceding siblings ...)
2021-10-13 18:22 ` [PULL 04/15] target/ppc: " Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 06/15] target/hexagon: Implement cpu_mmu_index Richard Henderson
` (10 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé, David Hildenbrand
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/s390x/tcg/mem_helper.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c
index 75f6735545..e64d1bc725 100644
--- a/target/s390x/tcg/mem_helper.c
+++ b/target/s390x/tcg/mem_helper.c
@@ -1811,7 +1811,7 @@ void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr,
assert(HAVE_CMPXCHG128);
mem_idx = cpu_mmu_index(env, false);
- oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+ oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
oldv = cpu_atomic_cmpxchgo_be_mmu(env, addr, cmpv, newv, oi, ra);
fail = !int128_eq(oldv, cmpv);
@@ -1940,7 +1940,7 @@ static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra);
cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra);
} else if (HAVE_CMPXCHG128) {
- MemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx);
+ MemOpIdx oi = make_memop_idx(MO_TE | MO_128 | MO_ALIGN, mem_idx);
ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra);
cc = !int128_eq(ov, cv);
} else {
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 06/15] target/hexagon: Implement cpu_mmu_index
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (4 preceding siblings ...)
2021-10-13 18:22 ` [PULL 05/15] target/s390x: " Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 07/15] accel/tcg: Add cpu_{ld,st}*_mmu interfaces Richard Henderson
` (9 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Taylor Simpson, Philippe Mathieu-Daudé
The function is trivial for user-only, but still must be present.
Reviewed-by: Taylor Simpson <tsimpson@quicinc.com>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/hexagon/cpu.h | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/target/hexagon/cpu.h b/target/hexagon/cpu.h
index f7d043865b..f90c187888 100644
--- a/target/hexagon/cpu.h
+++ b/target/hexagon/cpu.h
@@ -141,6 +141,15 @@ static inline void cpu_get_tb_cpu_state(CPUHexagonState *env, target_ulong *pc,
#endif
}
+static inline int cpu_mmu_index(CPUHexagonState *env, bool ifetch)
+{
+#ifdef CONFIG_USER_ONLY
+ return MMU_USER_IDX;
+#else
+#error System mode not supported on Hexagon yet
+#endif
+}
+
typedef struct CPUHexagonState CPUArchState;
typedef HexagonCPU ArchCPU;
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 07/15] accel/tcg: Add cpu_{ld,st}*_mmu interfaces
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (5 preceding siblings ...)
2021-10-13 18:22 ` [PULL 06/15] target/hexagon: Implement cpu_mmu_index Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 08/15] accel/tcg: Move cpu_atomic decls to exec/cpu_ldst.h Richard Henderson
` (8 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Peter Maydell
These functions are much closer to the softmmu helper
functions, in that they take the complete MemOpIdx,
and from that they may enforce required alignment.
The previous cpu_ldst.h functions did not have alignment info,
and so did not enforce it. Retain this by adding MO_UNALN to
the MemOp that we create in calling the new functions.
Note that we are not yet enforcing alignment for user-only,
but we now have the information with which to do so.
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
docs/devel/loads-stores.rst | 52 ++++-
include/exec/cpu_ldst.h | 245 ++++++++--------------
accel/tcg/cputlb.c | 392 ++++++++++++------------------------
accel/tcg/user-exec.c | 385 +++++++++++++++--------------------
accel/tcg/ldst_common.c.inc | 307 ++++++++++++++++++++++++++++
5 files changed, 717 insertions(+), 664 deletions(-)
create mode 100644 accel/tcg/ldst_common.c.inc
diff --git a/docs/devel/loads-stores.rst b/docs/devel/loads-stores.rst
index 568274baec..8f0035c821 100644
--- a/docs/devel/loads-stores.rst
+++ b/docs/devel/loads-stores.rst
@@ -68,15 +68,19 @@ Regexes for git grep
- ``\<ldn_\([hbl]e\)?_p\>``
- ``\<stn_\([hbl]e\)?_p\>``
-``cpu_{ld,st}*_mmuidx_ra``
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+``cpu_{ld,st}*_mmu``
+~~~~~~~~~~~~~~~~~~~~
-These functions operate on a guest virtual address plus a context,
-known as a "mmu index" or ``mmuidx``, which controls how that virtual
-address is translated. The meaning of the indexes are target specific,
-but specifying a particular index might be necessary if, for instance,
-the helper requires an "always as non-privileged" access rather that
-the default access for the current state of the guest CPU.
+These functions operate on a guest virtual address, plus a context
+known as a "mmu index" which controls how that virtual address is
+translated, plus a ``MemOp`` which contains alignment requirements
+among other things. The ``MemOp`` and mmu index are combined into
+a single argument of type ``MemOpIdx``.
+
+The meaning of the indexes are target specific, but specifying a
+particular index might be necessary if, for instance, the helper
+requires a "always as non-privileged" access rather than the
+default access for the current state of the guest CPU.
These functions may cause a guest CPU exception to be taken
(e.g. for an alignment fault or MMU fault) which will result in
@@ -99,6 +103,35 @@ function, which is a return address into the generated code [#gpc]_.
Function names follow the pattern:
+load: ``cpu_ld{size}{end}_mmu(env, ptr, oi, retaddr)``
+
+store: ``cpu_st{size}{end}_mmu(env, ptr, val, oi, retaddr)``
+
+``size``
+ - ``b`` : 8 bits
+ - ``w`` : 16 bits
+ - ``l`` : 32 bits
+ - ``q`` : 64 bits
+
+``end``
+ - (empty) : for target endian, or 8 bit sizes
+ - ``_be`` : big endian
+ - ``_le`` : little endian
+
+Regexes for git grep:
+ - ``\<cpu_ld[bwlq](_[bl]e)\?_mmu\>``
+ - ``\<cpu_st[bwlq](_[bl]e)\?_mmu\>``
+
+
+``cpu_{ld,st}*_mmuidx_ra``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These functions work like the ``cpu_{ld,st}_mmu`` functions except
+that the ``mmuidx`` parameter is not combined with a ``MemOp``,
+and therefore there is no required alignment supplied or enforced.
+
+Function names follow the pattern:
+
load: ``cpu_ld{sign}{size}{end}_mmuidx_ra(env, ptr, mmuidx, retaddr)``
store: ``cpu_st{size}{end}_mmuidx_ra(env, ptr, val, mmuidx, retaddr)``
@@ -132,7 +165,8 @@ of the guest CPU, as determined by ``cpu_mmu_index(env, false)``.
These are generally the preferred way to do accesses by guest
virtual address from helper functions, unless the access should
-be performed with a context other than the default.
+be performed with a context other than the default, or alignment
+should be enforced for the access.
Function names follow the pattern:
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index ce6ce82618..a4dad0772f 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -28,10 +28,12 @@
* load: cpu_ld{sign}{size}{end}_{mmusuffix}(env, ptr)
* cpu_ld{sign}{size}{end}_{mmusuffix}_ra(env, ptr, retaddr)
* cpu_ld{sign}{size}{end}_mmuidx_ra(env, ptr, mmu_idx, retaddr)
+ * cpu_ld{sign}{size}{end}_mmu(env, ptr, oi, retaddr)
*
* store: cpu_st{size}{end}_{mmusuffix}(env, ptr, val)
* cpu_st{size}{end}_{mmusuffix}_ra(env, ptr, val, retaddr)
* cpu_st{size}{end}_mmuidx_ra(env, ptr, val, mmu_idx, retaddr)
+ * cpu_st{size}{end}_mmu(env, ptr, val, oi, retaddr)
*
* sign is:
* (empty): for 32 and 64 bit sizes
@@ -53,10 +55,15 @@
* The "mmuidx" suffix carries an extra mmu_idx argument that specifies
* the index to use; the "data" and "code" suffixes take the index from
* cpu_mmu_index().
+ *
+ * The "mmu" suffix carries the full MemOpIdx, with both mmu_idx and the
+ * MemOp including alignment requirements. The alignment will be enforced.
*/
#ifndef CPU_LDST_H
#define CPU_LDST_H
+#include "exec/memopidx.h"
+
#if defined(CONFIG_USER_ONLY)
/* sparc32plus has 64bit long but 32bit space address
* this can make bad result with g2h() and h2g()
@@ -118,12 +125,10 @@ typedef target_ulong abi_ptr;
uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr);
int cpu_ldsb_data(CPUArchState *env, abi_ptr ptr);
-
uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr);
int cpu_ldsw_be_data(CPUArchState *env, abi_ptr ptr);
uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr);
uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr);
-
uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr);
int cpu_ldsw_le_data(CPUArchState *env, abi_ptr ptr);
uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr);
@@ -131,37 +136,31 @@ uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr ptr);
uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
-
uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
-
uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t ra);
void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
-
void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val);
-
void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val);
void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val);
void cpu_stb_data_ra(CPUArchState *env, abi_ptr ptr,
uint32_t val, uintptr_t ra);
-
void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr ptr,
uint32_t val, uintptr_t ra);
void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr ptr,
uint32_t val, uintptr_t ra);
void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr ptr,
uint64_t val, uintptr_t ra);
-
void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr ptr,
uint32_t val, uintptr_t ra);
void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr ptr,
@@ -169,6 +168,71 @@ void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr ptr,
void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr ptr,
uint64_t val, uintptr_t ra);
+uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr,
+ int mmu_idx, uintptr_t ra);
+
+void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ int mmu_idx, uintptr_t ra);
+void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ int mmu_idx, uintptr_t ra);
+void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ int mmu_idx, uintptr_t ra);
+void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint64_t val,
+ int mmu_idx, uintptr_t ra);
+void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ int mmu_idx, uintptr_t ra);
+void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ int mmu_idx, uintptr_t ra);
+void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr ptr, uint64_t val,
+ int mmu_idx, uintptr_t ra);
+
+uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr ptr, MemOpIdx oi, uintptr_t ra);
+uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr ptr,
+ MemOpIdx oi, uintptr_t ra);
+uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr ptr,
+ MemOpIdx oi, uintptr_t ra);
+uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr ptr,
+ MemOpIdx oi, uintptr_t ra);
+uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr ptr,
+ MemOpIdx oi, uintptr_t ra);
+uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr ptr,
+ MemOpIdx oi, uintptr_t ra);
+uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr ptr,
+ MemOpIdx oi, uintptr_t ra);
+
+void cpu_stb_mmu(CPUArchState *env, abi_ptr ptr, uint8_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stw_be_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stl_be_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stq_be_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stw_le_mmu(CPUArchState *env, abi_ptr ptr, uint16_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stl_le_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
+ MemOpIdx oi, uintptr_t ra);
+void cpu_stq_le_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
+ MemOpIdx oi, uintptr_t ra);
+
#if defined(CONFIG_USER_ONLY)
extern __thread uintptr_t helper_retaddr;
@@ -193,119 +257,6 @@ static inline void clear_helper_retaddr(void)
helper_retaddr = 0;
}
-/*
- * Provide the same *_mmuidx_ra interface as for softmmu.
- * The mmu_idx argument is ignored.
- */
-
-static inline uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldub_data_ra(env, addr, ra);
-}
-
-static inline int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldsb_data_ra(env, addr, ra);
-}
-
-static inline uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_lduw_be_data_ra(env, addr, ra);
-}
-
-static inline int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldsw_be_data_ra(env, addr, ra);
-}
-
-static inline uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldl_be_data_ra(env, addr, ra);
-}
-
-static inline uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldq_be_data_ra(env, addr, ra);
-}
-
-static inline uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_lduw_le_data_ra(env, addr, ra);
-}
-
-static inline int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldsw_le_data_ra(env, addr, ra);
-}
-
-static inline uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldl_le_data_ra(env, addr, ra);
-}
-
-static inline uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_ldq_le_data_ra(env, addr, ra);
-}
-
-static inline void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, int mmu_idx, uintptr_t ra)
-{
- cpu_stb_data_ra(env, addr, val, ra);
-}
-
-static inline void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, int mmu_idx,
- uintptr_t ra)
-{
- cpu_stw_be_data_ra(env, addr, val, ra);
-}
-
-static inline void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, int mmu_idx,
- uintptr_t ra)
-{
- cpu_stl_be_data_ra(env, addr, val, ra);
-}
-
-static inline void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint64_t val, int mmu_idx,
- uintptr_t ra)
-{
- cpu_stq_be_data_ra(env, addr, val, ra);
-}
-
-static inline void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, int mmu_idx,
- uintptr_t ra)
-{
- cpu_stw_le_data_ra(env, addr, val, ra);
-}
-
-static inline void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint32_t val, int mmu_idx,
- uintptr_t ra)
-{
- cpu_stl_le_data_ra(env, addr, val, ra);
-}
-
-static inline void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- uint64_t val, int mmu_idx,
- uintptr_t ra)
-{
- cpu_stq_le_data_ra(env, addr, val, ra);
-}
-
#else
/* Needed for TCG_OVERSIZED_GUEST */
@@ -336,46 +287,6 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx,
return &env_tlb(env)->f[mmu_idx].table[tlb_index(env, mmu_idx, addr)];
}
-uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-
-uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-
-uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra);
-
-void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr);
-
-void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr);
-void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr);
-void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
- int mmu_idx, uintptr_t retaddr);
-
-void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr);
-void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr);
-void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
- int mmu_idx, uintptr_t retaddr);
-
#endif /* defined(CONFIG_USER_ONLY) */
#ifdef TARGET_WORDS_BIGENDIAN
@@ -391,6 +302,9 @@ void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
# define cpu_ldsw_mmuidx_ra cpu_ldsw_be_mmuidx_ra
# define cpu_ldl_mmuidx_ra cpu_ldl_be_mmuidx_ra
# define cpu_ldq_mmuidx_ra cpu_ldq_be_mmuidx_ra
+# define cpu_ldw_mmu cpu_ldw_be_mmu
+# define cpu_ldl_mmu cpu_ldl_be_mmu
+# define cpu_ldq_mmu cpu_ldq_be_mmu
# define cpu_stw_data cpu_stw_be_data
# define cpu_stl_data cpu_stl_be_data
# define cpu_stq_data cpu_stq_be_data
@@ -400,6 +314,9 @@ void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
# define cpu_stw_mmuidx_ra cpu_stw_be_mmuidx_ra
# define cpu_stl_mmuidx_ra cpu_stl_be_mmuidx_ra
# define cpu_stq_mmuidx_ra cpu_stq_be_mmuidx_ra
+# define cpu_stw_mmu cpu_stw_be_mmu
+# define cpu_stl_mmu cpu_stl_be_mmu
+# define cpu_stq_mmu cpu_stq_be_mmu
#else
# define cpu_lduw_data cpu_lduw_le_data
# define cpu_ldsw_data cpu_ldsw_le_data
@@ -413,6 +330,9 @@ void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
# define cpu_ldsw_mmuidx_ra cpu_ldsw_le_mmuidx_ra
# define cpu_ldl_mmuidx_ra cpu_ldl_le_mmuidx_ra
# define cpu_ldq_mmuidx_ra cpu_ldq_le_mmuidx_ra
+# define cpu_ldw_mmu cpu_ldw_le_mmu
+# define cpu_ldl_mmu cpu_ldl_le_mmu
+# define cpu_ldq_mmu cpu_ldq_le_mmu
# define cpu_stw_data cpu_stw_le_data
# define cpu_stl_data cpu_stl_le_data
# define cpu_stq_data cpu_stq_le_data
@@ -422,6 +342,9 @@ void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
# define cpu_stw_mmuidx_ra cpu_stw_le_mmuidx_ra
# define cpu_stl_mmuidx_ra cpu_stl_le_mmuidx_ra
# define cpu_stq_mmuidx_ra cpu_stq_le_mmuidx_ra
+# define cpu_stw_mmu cpu_stw_le_mmu
+# define cpu_stl_mmu cpu_stl_le_mmu
+# define cpu_stq_mmu cpu_stq_le_mmu
#endif
uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr);
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 46140ccff3..b350cafa3d 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -1839,6 +1839,25 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
cpu_loop_exit_atomic(env_cpu(env), retaddr);
}
+/*
+ * Verify that we have passed the correct MemOp to the correct function.
+ *
+ * In the case of the helper_*_mmu functions, we will have done this by
+ * using the MemOp to look up the helper during code generation.
+ *
+ * In the case of the cpu_*_mmu functions, this is up to the caller.
+ * We could present one function to target code, and dispatch based on
+ * the MemOp, but so far we have worked hard to avoid an indirect function
+ * call along the memory path.
+ */
+static void validate_memop(MemOpIdx oi, MemOp expected)
+{
+#ifdef CONFIG_DEBUG_TCG
+ MemOp have = get_memop(oi) & (MO_SIZE | MO_BSWAP);
+ assert(have == expected);
+#endif
+}
+
/*
* Load Helpers
*
@@ -1992,6 +2011,7 @@ load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi,
static uint64_t full_ldub_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_UB);
return load_helper(env, addr, oi, retaddr, MO_UB, false, full_ldub_mmu);
}
@@ -2004,6 +2024,7 @@ tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
static uint64_t full_le_lduw_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_LEUW);
return load_helper(env, addr, oi, retaddr, MO_LEUW, false,
full_le_lduw_mmu);
}
@@ -2017,6 +2038,7 @@ tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
static uint64_t full_be_lduw_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_BEUW);
return load_helper(env, addr, oi, retaddr, MO_BEUW, false,
full_be_lduw_mmu);
}
@@ -2030,6 +2052,7 @@ tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
static uint64_t full_le_ldul_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_LEUL);
return load_helper(env, addr, oi, retaddr, MO_LEUL, false,
full_le_ldul_mmu);
}
@@ -2043,6 +2066,7 @@ tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
static uint64_t full_be_ldul_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_BEUL);
return load_helper(env, addr, oi, retaddr, MO_BEUL, false,
full_be_ldul_mmu);
}
@@ -2056,6 +2080,7 @@ tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_LEQ);
return load_helper(env, addr, oi, retaddr, MO_LEQ, false,
helper_le_ldq_mmu);
}
@@ -2063,6 +2088,7 @@ uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_BEQ);
return load_helper(env, addr, oi, retaddr, MO_BEQ, false,
helper_be_ldq_mmu);
}
@@ -2108,186 +2134,56 @@ tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
*/
static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t retaddr,
- MemOp op, FullLoadHelper *full_load)
+ MemOpIdx oi, uintptr_t retaddr,
+ FullLoadHelper *full_load)
{
- MemOpIdx oi = make_memop_idx(op, mmu_idx);
uint64_t ret;
trace_guest_ld_before_exec(env_cpu(env), addr, oi);
-
ret = full_load(env, addr, oi, retaddr);
-
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
-
return ret;
}
-uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_UB, full_ldub_mmu);
+ return cpu_load_helper(env, addr, oi, ra, full_ldub_mmu);
}
-int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int8_t)cpu_ldub_mmuidx_ra(env, addr, mmu_idx, ra);
+ return cpu_load_helper(env, addr, oi, ra, full_be_lduw_mmu);
}
-uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_BEUW, full_be_lduw_mmu);
+ return cpu_load_helper(env, addr, oi, ra, full_be_ldul_mmu);
}
-int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int16_t)cpu_lduw_be_mmuidx_ra(env, addr, mmu_idx, ra);
+ return cpu_load_helper(env, addr, oi, MO_BEQ, helper_be_ldq_mmu);
}
-uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_BEUL, full_be_ldul_mmu);
+ return cpu_load_helper(env, addr, oi, ra, full_le_lduw_mmu);
}
-uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_BEQ, helper_be_ldq_mmu);
+ return cpu_load_helper(env, addr, oi, ra, full_le_ldul_mmu);
}
-uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
+uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_LEUW, full_le_lduw_mmu);
-}
-
-int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return (int16_t)cpu_lduw_le_mmuidx_ra(env, addr, mmu_idx, ra);
-}
-
-uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_LEUL, full_le_ldul_mmu);
-}
-
-uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
- int mmu_idx, uintptr_t ra)
-{
- return cpu_load_helper(env, addr, mmu_idx, ra, MO_LEQ, helper_le_ldq_mmu);
-}
-
-uint32_t cpu_ldub_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_ldub_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-int cpu_ldsb_data_ra(CPUArchState *env, target_ulong ptr, uintptr_t retaddr)
-{
- return cpu_ldsb_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint32_t cpu_lduw_be_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_lduw_be_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-int cpu_ldsw_be_data_ra(CPUArchState *env, target_ulong ptr, uintptr_t retaddr)
-{
- return cpu_ldsw_be_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint32_t cpu_ldl_be_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_ldl_be_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint64_t cpu_ldq_be_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_ldq_be_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint32_t cpu_lduw_le_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_lduw_le_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-int cpu_ldsw_le_data_ra(CPUArchState *env, target_ulong ptr, uintptr_t retaddr)
-{
- return cpu_ldsw_le_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint32_t cpu_ldl_le_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_ldl_le_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint64_t cpu_ldq_le_data_ra(CPUArchState *env, target_ulong ptr,
- uintptr_t retaddr)
-{
- return cpu_ldq_le_mmuidx_ra(env, ptr, cpu_mmu_index(env, false), retaddr);
-}
-
-uint32_t cpu_ldub_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldub_data_ra(env, ptr, 0);
-}
-
-int cpu_ldsb_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldsb_data_ra(env, ptr, 0);
-}
-
-uint32_t cpu_lduw_be_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_lduw_be_data_ra(env, ptr, 0);
-}
-
-int cpu_ldsw_be_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldsw_be_data_ra(env, ptr, 0);
-}
-
-uint32_t cpu_ldl_be_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldl_be_data_ra(env, ptr, 0);
-}
-
-uint64_t cpu_ldq_be_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldq_be_data_ra(env, ptr, 0);
-}
-
-uint32_t cpu_lduw_le_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_lduw_le_data_ra(env, ptr, 0);
-}
-
-int cpu_ldsw_le_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldsw_le_data_ra(env, ptr, 0);
-}
-
-uint32_t cpu_ldl_le_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldl_le_data_ra(env, ptr, 0);
-}
-
-uint64_t cpu_ldq_le_data(CPUArchState *env, target_ulong ptr)
-{
- return cpu_ldq_le_data_ra(env, ptr, 0);
+ return cpu_load_helper(env, addr, oi, ra, helper_le_ldq_mmu);
}
/*
@@ -2324,6 +2220,9 @@ store_memop(void *haddr, uint64_t val, MemOp op)
}
}
+static void full_stb_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+
static void __attribute__((noinline))
store_helper_unaligned(CPUArchState *env, target_ulong addr, uint64_t val,
uintptr_t retaddr, size_t size, uintptr_t mmu_idx,
@@ -2387,13 +2286,13 @@ store_helper_unaligned(CPUArchState *env, target_ulong addr, uint64_t val,
for (i = 0; i < size; ++i) {
/* Big-endian extract. */
uint8_t val8 = val >> (((size - 1) * 8) - (i * 8));
- helper_ret_stb_mmu(env, addr + i, val8, oi, retaddr);
+ full_stb_mmu(env, addr + i, val8, oi, retaddr);
}
} else {
for (i = 0; i < size; ++i) {
/* Little-endian extract. */
uint8_t val8 = val >> (i * 8);
- helper_ret_stb_mmu(env, addr + i, val8, oi, retaddr);
+ full_stb_mmu(env, addr + i, val8, oi, retaddr);
}
}
}
@@ -2496,46 +2395,83 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
store_memop(haddr, val, op);
}
-void __attribute__((noinline))
-helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
- MemOpIdx oi, uintptr_t retaddr)
+static void __attribute__((noinline))
+full_stb_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_UB);
store_helper(env, addr, val, oi, retaddr, MO_UB);
}
+void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
+ MemOpIdx oi, uintptr_t retaddr)
+{
+ full_stb_mmu(env, addr, val, oi, retaddr);
+}
+
+static void full_le_stw_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
+{
+ validate_memop(oi, MO_LEUW);
+ store_helper(env, addr, val, oi, retaddr, MO_LEUW);
+}
+
void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
MemOpIdx oi, uintptr_t retaddr)
{
- store_helper(env, addr, val, oi, retaddr, MO_LEUW);
+ full_le_stw_mmu(env, addr, val, oi, retaddr);
+}
+
+static void full_be_stw_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
+{
+ validate_memop(oi, MO_BEUW);
+ store_helper(env, addr, val, oi, retaddr, MO_BEUW);
}
void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
MemOpIdx oi, uintptr_t retaddr)
{
- store_helper(env, addr, val, oi, retaddr, MO_BEUW);
+ full_be_stw_mmu(env, addr, val, oi, retaddr);
+}
+
+static void full_le_stl_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
+{
+ validate_memop(oi, MO_LEUL);
+ store_helper(env, addr, val, oi, retaddr, MO_LEUL);
}
void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
MemOpIdx oi, uintptr_t retaddr)
{
- store_helper(env, addr, val, oi, retaddr, MO_LEUL);
+ full_le_stl_mmu(env, addr, val, oi, retaddr);
+}
+
+static void full_be_stl_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
+{
+ validate_memop(oi, MO_BEUL);
+ store_helper(env, addr, val, oi, retaddr, MO_BEUL);
}
void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
MemOpIdx oi, uintptr_t retaddr)
{
- store_helper(env, addr, val, oi, retaddr, MO_BEUL);
+ full_be_stl_mmu(env, addr, val, oi, retaddr);
}
void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_LEQ);
store_helper(env, addr, val, oi, retaddr, MO_LEQ);
}
void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
MemOpIdx oi, uintptr_t retaddr)
{
+ validate_memop(oi, MO_BEQ);
store_helper(env, addr, val, oi, retaddr, MO_BEQ);
}
@@ -2543,137 +2479,61 @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
* Store Helpers for cpu_ldst.h
*/
-static inline void QEMU_ALWAYS_INLINE
-cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
- int mmu_idx, uintptr_t retaddr, MemOp op)
+typedef void FullStoreHelper(CPUArchState *env, target_ulong addr,
+ uint64_t val, MemOpIdx oi, uintptr_t retaddr);
+
+static inline void cpu_store_helper(CPUArchState *env, target_ulong addr,
+ uint64_t val, MemOpIdx oi, uintptr_t ra,
+ FullStoreHelper *full_store)
{
- MemOpIdx oi = make_memop_idx(op, mmu_idx);
-
trace_guest_st_before_exec(env_cpu(env), addr, oi);
-
- store_helper(env, addr, val, oi, retaddr, op);
-
+ full_store(env, addr, val, oi, ra);
qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-void cpu_stb_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_UB);
+ cpu_store_helper(env, addr, val, oi, retaddr, full_stb_mmu);
}
-void cpu_stw_be_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stw_be_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_BEUW);
+ cpu_store_helper(env, addr, val, oi, retaddr, full_be_stw_mmu);
}
-void cpu_stl_be_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stl_be_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_BEUL);
+ cpu_store_helper(env, addr, val, oi, retaddr, full_be_stl_mmu);
}
-void cpu_stq_be_mmuidx_ra(CPUArchState *env, target_ulong addr, uint64_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stq_be_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_BEQ);
+ cpu_store_helper(env, addr, val, oi, retaddr, helper_be_stq_mmu);
}
-void cpu_stw_le_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stw_le_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_LEUW);
+ cpu_store_helper(env, addr, val, oi, retaddr, full_le_stw_mmu);
}
-void cpu_stl_le_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stl_le_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_LEUL);
+ cpu_store_helper(env, addr, val, oi, retaddr, full_le_stl_mmu);
}
-void cpu_stq_le_mmuidx_ra(CPUArchState *env, target_ulong addr, uint64_t val,
- int mmu_idx, uintptr_t retaddr)
+void cpu_stq_le_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr)
{
- cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_LEQ);
+ cpu_store_helper(env, addr, val, oi, retaddr, helper_le_stq_mmu);
}
-void cpu_stb_data_ra(CPUArchState *env, target_ulong ptr,
- uint32_t val, uintptr_t retaddr)
-{
- cpu_stb_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stw_be_data_ra(CPUArchState *env, target_ulong ptr,
- uint32_t val, uintptr_t retaddr)
-{
- cpu_stw_be_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stl_be_data_ra(CPUArchState *env, target_ulong ptr,
- uint32_t val, uintptr_t retaddr)
-{
- cpu_stl_be_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stq_be_data_ra(CPUArchState *env, target_ulong ptr,
- uint64_t val, uintptr_t retaddr)
-{
- cpu_stq_be_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stw_le_data_ra(CPUArchState *env, target_ulong ptr,
- uint32_t val, uintptr_t retaddr)
-{
- cpu_stw_le_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stl_le_data_ra(CPUArchState *env, target_ulong ptr,
- uint32_t val, uintptr_t retaddr)
-{
- cpu_stl_le_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stq_le_data_ra(CPUArchState *env, target_ulong ptr,
- uint64_t val, uintptr_t retaddr)
-{
- cpu_stq_le_mmuidx_ra(env, ptr, val, cpu_mmu_index(env, false), retaddr);
-}
-
-void cpu_stb_data(CPUArchState *env, target_ulong ptr, uint32_t val)
-{
- cpu_stb_data_ra(env, ptr, val, 0);
-}
-
-void cpu_stw_be_data(CPUArchState *env, target_ulong ptr, uint32_t val)
-{
- cpu_stw_be_data_ra(env, ptr, val, 0);
-}
-
-void cpu_stl_be_data(CPUArchState *env, target_ulong ptr, uint32_t val)
-{
- cpu_stl_be_data_ra(env, ptr, val, 0);
-}
-
-void cpu_stq_be_data(CPUArchState *env, target_ulong ptr, uint64_t val)
-{
- cpu_stq_be_data_ra(env, ptr, val, 0);
-}
-
-void cpu_stw_le_data(CPUArchState *env, target_ulong ptr, uint32_t val)
-{
- cpu_stw_le_data_ra(env, ptr, val, 0);
-}
-
-void cpu_stl_le_data(CPUArchState *env, target_ulong ptr, uint32_t val)
-{
- cpu_stl_le_data_ra(env, ptr, val, 0);
-}
-
-void cpu_stq_le_data(CPUArchState *env, target_ulong ptr, uint64_t val)
-{
- cpu_stq_le_data_ra(env, ptr, val, 0);
-}
+#include "ldst_common.c.inc"
/*
* First set of functions passes in OI and RETADDR.
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index 65d3c9b286..e6bb29b42d 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -886,300 +886,227 @@ int cpu_signal_handler(int host_signum, void *pinfo,
/* The softmmu versions of these helpers are in cputlb.c. */
-uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr ptr)
+/*
+ * Verify that we have passed the correct MemOp to the correct function.
+ *
+ * We could present one function to target code, and dispatch based on
+ * the MemOp, but so far we have worked hard to avoid an indirect function
+ * call along the memory path.
+ */
+static void validate_memop(MemOpIdx oi, MemOp expected)
{
- MemOpIdx oi = make_memop_idx(MO_UB, MMU_USER_IDX);
- uint32_t ret;
+#ifdef CONFIG_DEBUG_TCG
+ MemOp have = get_memop(oi) & (MO_SIZE | MO_BSWAP);
+ assert(have == expected);
+#endif
+}
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = ldub_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
+static void *cpu_mmu_lookup(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t ra, MMUAccessType type)
+{
+ void *ret;
+
+ /* TODO: Enforce guest required alignment. */
+
+ ret = g2h(env_cpu(env), addr);
+ set_helper_retaddr(ra);
return ret;
}
-int cpu_ldsb_data(CPUArchState *env, abi_ptr ptr)
+uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int8_t)cpu_ldub_data(env, ptr);
-}
+ void *haddr;
+ uint8_t ret;
-uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr ptr)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUW, MMU_USER_IDX);
- uint32_t ret;
-
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = lduw_be_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
+ validate_memop(oi, MO_UB);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = ldub_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
return ret;
}
-int cpu_ldsw_be_data(CPUArchState *env, abi_ptr ptr)
+uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int16_t)cpu_lduw_be_data(env, ptr);
-}
+ void *haddr;
+ uint16_t ret;
-uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr ptr)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUL, MMU_USER_IDX);
- uint32_t ret;
-
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = ldl_be_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
+ validate_memop(oi, MO_BEUW);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = lduw_be_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
return ret;
}
-uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr ptr)
+uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOpIdx oi = make_memop_idx(MO_BEQ, MMU_USER_IDX);
+ void *haddr;
+ uint32_t ret;
+
+ validate_memop(oi, MO_BEUL);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = ldl_be_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
+ return ret;
+}
+
+uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
+{
+ void *haddr;
uint64_t ret;
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = ldq_be_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
+ validate_memop(oi, MO_BEQ);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = ldq_be_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
return ret;
}
-uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr ptr)
+uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- MemOpIdx oi = make_memop_idx(MO_LEUW, MMU_USER_IDX);
+ void *haddr;
+ uint16_t ret;
+
+ validate_memop(oi, MO_LEUW);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = lduw_le_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
+ return ret;
+}
+
+uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
+{
+ void *haddr;
uint32_t ret;
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = lduw_le_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
+ validate_memop(oi, MO_LEUL);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = ldl_le_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
return ret;
}
-int cpu_ldsw_le_data(CPUArchState *env, abi_ptr ptr)
+uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int16_t)cpu_lduw_le_data(env, ptr);
-}
-
-uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr ptr)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUL, MMU_USER_IDX);
- uint32_t ret;
-
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = ldl_le_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
- return ret;
-}
-
-uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr ptr)
-{
- MemOpIdx oi = make_memop_idx(MO_LEQ, MMU_USER_IDX);
+ void *haddr;
uint64_t ret;
- trace_guest_ld_before_exec(env_cpu(env), ptr, oi);
- ret = ldq_le_p(g2h(env_cpu(env), ptr));
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_R);
+ validate_memop(oi, MO_LEQ);
+ trace_guest_ld_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD);
+ ret = ldq_le_p(haddr);
+ clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
return ret;
}
-uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- uint32_t ret;
+ void *haddr;
- set_helper_retaddr(retaddr);
- ret = cpu_ldub_data(env, ptr);
+ validate_memop(oi, MO_UB);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stb_p(haddr, val);
clear_helper_retaddr();
- return ret;
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stw_be_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int8_t)cpu_ldub_data_ra(env, ptr, retaddr);
-}
+ void *haddr;
-uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
-{
- uint32_t ret;
-
- set_helper_retaddr(retaddr);
- ret = cpu_lduw_be_data(env, ptr);
+ validate_memop(oi, MO_BEUW);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stw_be_p(haddr, val);
clear_helper_retaddr();
- return ret;
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stl_be_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int16_t)cpu_lduw_be_data_ra(env, ptr, retaddr);
-}
+ void *haddr;
-uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
-{
- uint32_t ret;
-
- set_helper_retaddr(retaddr);
- ret = cpu_ldl_be_data(env, ptr);
+ validate_memop(oi, MO_BEUL);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stl_be_p(haddr, val);
clear_helper_retaddr();
- return ret;
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stq_be_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- uint64_t ret;
+ void *haddr;
- set_helper_retaddr(retaddr);
- ret = cpu_ldq_be_data(env, ptr);
+ validate_memop(oi, MO_BEQ);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stq_be_p(haddr, val);
clear_helper_retaddr();
- return ret;
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stw_le_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- uint32_t ret;
+ void *haddr;
- set_helper_retaddr(retaddr);
- ret = cpu_lduw_le_data(env, ptr);
+ validate_memop(oi, MO_LEUW);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stw_le_p(haddr, val);
clear_helper_retaddr();
- return ret;
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stl_le_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- return (int16_t)cpu_lduw_le_data_ra(env, ptr, retaddr);
-}
+ void *haddr;
-uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
-{
- uint32_t ret;
-
- set_helper_retaddr(retaddr);
- ret = cpu_ldl_le_data(env, ptr);
+ validate_memop(oi, MO_LEUL);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stl_le_p(haddr, val);
clear_helper_retaddr();
- return ret;
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
-uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr ptr, uintptr_t retaddr)
+void cpu_stq_le_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
+ MemOpIdx oi, uintptr_t ra)
{
- uint64_t ret;
+ void *haddr;
- set_helper_retaddr(retaddr);
- ret = cpu_ldq_le_data(env, ptr);
- clear_helper_retaddr();
- return ret;
-}
-
-void cpu_stb_data(CPUArchState *env, abi_ptr ptr, uint32_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_UB, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stb_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stw_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUW, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stw_be_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stl_be_data(CPUArchState *env, abi_ptr ptr, uint32_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_BEUL, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stl_be_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stq_be_data(CPUArchState *env, abi_ptr ptr, uint64_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_BEQ, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stq_be_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stw_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUW, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stw_le_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stl_le_data(CPUArchState *env, abi_ptr ptr, uint32_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_LEUL, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stl_le_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stq_le_data(CPUArchState *env, abi_ptr ptr, uint64_t val)
-{
- MemOpIdx oi = make_memop_idx(MO_LEQ, MMU_USER_IDX);
-
- trace_guest_st_before_exec(env_cpu(env), ptr, oi);
- stq_le_p(g2h(env_cpu(env), ptr), val);
- qemu_plugin_vcpu_mem_cb(env_cpu(env), ptr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stb_data_ra(CPUArchState *env, abi_ptr ptr,
- uint32_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stb_data(env, ptr, val);
- clear_helper_retaddr();
-}
-
-void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr ptr,
- uint32_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stw_be_data(env, ptr, val);
- clear_helper_retaddr();
-}
-
-void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr ptr,
- uint32_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stl_be_data(env, ptr, val);
- clear_helper_retaddr();
-}
-
-void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr ptr,
- uint64_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stq_be_data(env, ptr, val);
- clear_helper_retaddr();
-}
-
-void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr ptr,
- uint32_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stw_le_data(env, ptr, val);
- clear_helper_retaddr();
-}
-
-void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr ptr,
- uint32_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stl_le_data(env, ptr, val);
- clear_helper_retaddr();
-}
-
-void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr ptr,
- uint64_t val, uintptr_t retaddr)
-{
- set_helper_retaddr(retaddr);
- cpu_stq_le_data(env, ptr, val);
+ validate_memop(oi, MO_LEQ);
+ trace_guest_st_before_exec(env_cpu(env), addr, oi);
+ haddr = cpu_mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE);
+ stq_le_p(haddr, val);
clear_helper_retaddr();
+ qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
}
uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr ptr)
@@ -1222,6 +1149,8 @@ uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr ptr)
return ret;
}
+#include "ldst_common.c.inc"
+
/*
* Do not allow unaligned operations to proceed. Return the host address.
*
diff --git a/accel/tcg/ldst_common.c.inc b/accel/tcg/ldst_common.c.inc
new file mode 100644
index 0000000000..bfefb275e7
--- /dev/null
+++ b/accel/tcg/ldst_common.c.inc
@@ -0,0 +1,307 @@
+/*
+ * Routines common to user and system emulation of load/store.
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
+ return cpu_ldb_mmu(env, addr, oi, ra);
+}
+
+int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ return (int8_t)cpu_ldub_mmuidx_ra(env, addr, mmu_idx, ra);
+}
+
+uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
+ return cpu_ldw_be_mmu(env, addr, oi, ra);
+}
+
+int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ return (int16_t)cpu_lduw_be_mmuidx_ra(env, addr, mmu_idx, ra);
+}
+
+uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
+ return cpu_ldl_be_mmu(env, addr, oi, ra);
+}
+
+uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_BEQ | MO_UNALN, mmu_idx);
+ return cpu_ldq_be_mmu(env, addr, oi, ra);
+}
+
+uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
+ return cpu_ldw_le_mmu(env, addr, oi, ra);
+}
+
+int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ return (int16_t)cpu_lduw_le_mmuidx_ra(env, addr, mmu_idx, ra);
+}
+
+uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
+ return cpu_ldl_le_mmu(env, addr, oi, ra);
+}
+
+uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_LEQ | MO_UNALN, mmu_idx);
+ return cpu_ldq_le_mmu(env, addr, oi, ra);
+}
+
+void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
+ cpu_stb_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
+ cpu_stw_be_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
+ cpu_stl_be_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_BEQ | MO_UNALN, mmu_idx);
+ cpu_stq_be_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
+ cpu_stw_le_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
+ cpu_stl_le_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
+ int mmu_idx, uintptr_t ra)
+{
+ MemOpIdx oi = make_memop_idx(MO_LEQ | MO_UNALN, mmu_idx);
+ cpu_stq_le_mmu(env, addr, val, oi, ra);
+}
+
+/*--------------------------*/
+
+uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_ldub_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return (int8_t)cpu_ldub_data_ra(env, addr, ra);
+}
+
+uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_lduw_be_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return (int16_t)cpu_lduw_be_data_ra(env, addr, ra);
+}
+
+uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_ldl_be_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_ldq_be_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_lduw_le_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return (int16_t)cpu_lduw_le_data_ra(env, addr, ra);
+}
+
+uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_ldl_le_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+ return cpu_ldq_le_mmuidx_ra(env, addr, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stb_data_ra(CPUArchState *env, abi_ptr addr,
+ uint32_t val, uintptr_t ra)
+{
+ cpu_stb_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr addr,
+ uint32_t val, uintptr_t ra)
+{
+ cpu_stw_be_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr addr,
+ uint32_t val, uintptr_t ra)
+{
+ cpu_stl_be_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr addr,
+ uint64_t val, uintptr_t ra)
+{
+ cpu_stq_be_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr addr,
+ uint32_t val, uintptr_t ra)
+{
+ cpu_stw_le_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr addr,
+ uint32_t val, uintptr_t ra)
+{
+ cpu_stl_le_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr addr,
+ uint64_t val, uintptr_t ra)
+{
+ cpu_stq_le_mmuidx_ra(env, addr, val, cpu_mmu_index(env, false), ra);
+}
+
+/*--------------------------*/
+
+uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_ldub_data_ra(env, addr, 0);
+}
+
+int cpu_ldsb_data(CPUArchState *env, abi_ptr addr)
+{
+ return (int8_t)cpu_ldub_data(env, addr);
+}
+
+uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_lduw_be_data_ra(env, addr, 0);
+}
+
+int cpu_ldsw_be_data(CPUArchState *env, abi_ptr addr)
+{
+ return (int16_t)cpu_lduw_be_data(env, addr);
+}
+
+uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_ldl_be_data_ra(env, addr, 0);
+}
+
+uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_ldq_be_data_ra(env, addr, 0);
+}
+
+uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_lduw_le_data_ra(env, addr, 0);
+}
+
+int cpu_ldsw_le_data(CPUArchState *env, abi_ptr addr)
+{
+ return (int16_t)cpu_lduw_le_data(env, addr);
+}
+
+uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_ldl_le_data_ra(env, addr, 0);
+}
+
+uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr addr)
+{
+ return cpu_ldq_le_data_ra(env, addr, 0);
+}
+
+void cpu_stb_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+ cpu_stb_data_ra(env, addr, val, 0);
+}
+
+void cpu_stw_be_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+ cpu_stw_be_data_ra(env, addr, val, 0);
+}
+
+void cpu_stl_be_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+ cpu_stl_be_data_ra(env, addr, val, 0);
+}
+
+void cpu_stq_be_data(CPUArchState *env, abi_ptr addr, uint64_t val)
+{
+ cpu_stq_be_data_ra(env, addr, val, 0);
+}
+
+void cpu_stw_le_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+ cpu_stw_le_data_ra(env, addr, val, 0);
+}
+
+void cpu_stl_le_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+ cpu_stl_le_data_ra(env, addr, val, 0);
+}
+
+void cpu_stq_le_data(CPUArchState *env, abi_ptr addr, uint64_t val)
+{
+ cpu_stq_le_data_ra(env, addr, val, 0);
+}
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 08/15] accel/tcg: Move cpu_atomic decls to exec/cpu_ldst.h
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (6 preceding siblings ...)
2021-10-13 18:22 ` [PULL 07/15] accel/tcg: Add cpu_{ld,st}*_mmu interfaces Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 09/15] target/mips: Use cpu_*_data_ra for msa load/store Richard Henderson
` (7 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé
The previous placement in tcg/tcg.h was not logical.
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
include/exec/cpu_ldst.h | 87 +++++++++++++++++++++++++++++++++++
include/tcg/tcg.h | 87 -----------------------------------
target/arm/helper-a64.c | 1 -
target/m68k/op_helper.c | 1 -
target/ppc/mem_helper.c | 1 -
target/s390x/tcg/mem_helper.c | 1 -
6 files changed, 87 insertions(+), 91 deletions(-)
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index a4dad0772f..a878fd0105 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -63,6 +63,7 @@
#define CPU_LDST_H
#include "exec/memopidx.h"
+#include "qemu/int128.h"
#if defined(CONFIG_USER_ONLY)
/* sparc32plus has 64bit long but 32bit space address
@@ -233,6 +234,92 @@ void cpu_stl_le_mmu(CPUArchState *env, abi_ptr ptr, uint32_t val,
void cpu_stq_le_mmu(CPUArchState *env, abi_ptr ptr, uint64_t val,
MemOpIdx oi, uintptr_t ra);
+uint32_t cpu_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
+ uint32_t cmpv, uint32_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+uint32_t cpu_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
+ uint32_t cmpv, uint32_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+uint32_t cpu_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
+ uint32_t cmpv, uint32_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+uint64_t cpu_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
+ uint64_t cmpv, uint64_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+uint32_t cpu_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
+ uint32_t cmpv, uint32_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+uint32_t cpu_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
+ uint32_t cmpv, uint32_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+uint64_t cpu_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
+ uint64_t cmpv, uint64_t newv,
+ MemOpIdx oi, uintptr_t retaddr);
+
+#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
+TYPE cpu_atomic_ ## NAME ## SUFFIX ## _mmu \
+ (CPUArchState *env, target_ulong addr, TYPE val, \
+ MemOpIdx oi, uintptr_t retaddr);
+
+#ifdef CONFIG_ATOMIC64
+#define GEN_ATOMIC_HELPER_ALL(NAME) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
+ GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
+ GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
+#else
+#define GEN_ATOMIC_HELPER_ALL(NAME) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
+ GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
+#endif
+
+GEN_ATOMIC_HELPER_ALL(fetch_add)
+GEN_ATOMIC_HELPER_ALL(fetch_sub)
+GEN_ATOMIC_HELPER_ALL(fetch_and)
+GEN_ATOMIC_HELPER_ALL(fetch_or)
+GEN_ATOMIC_HELPER_ALL(fetch_xor)
+GEN_ATOMIC_HELPER_ALL(fetch_smin)
+GEN_ATOMIC_HELPER_ALL(fetch_umin)
+GEN_ATOMIC_HELPER_ALL(fetch_smax)
+GEN_ATOMIC_HELPER_ALL(fetch_umax)
+
+GEN_ATOMIC_HELPER_ALL(add_fetch)
+GEN_ATOMIC_HELPER_ALL(sub_fetch)
+GEN_ATOMIC_HELPER_ALL(and_fetch)
+GEN_ATOMIC_HELPER_ALL(or_fetch)
+GEN_ATOMIC_HELPER_ALL(xor_fetch)
+GEN_ATOMIC_HELPER_ALL(smin_fetch)
+GEN_ATOMIC_HELPER_ALL(umin_fetch)
+GEN_ATOMIC_HELPER_ALL(smax_fetch)
+GEN_ATOMIC_HELPER_ALL(umax_fetch)
+
+GEN_ATOMIC_HELPER_ALL(xchg)
+
+#undef GEN_ATOMIC_HELPER_ALL
+#undef GEN_ATOMIC_HELPER
+
+Int128 cpu_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
+ Int128 cmpv, Int128 newv,
+ MemOpIdx oi, uintptr_t retaddr);
+Int128 cpu_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
+ Int128 cmpv, Int128 newv,
+ MemOpIdx oi, uintptr_t retaddr);
+
+Int128 cpu_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+Int128 cpu_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+void cpu_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
+ MemOpIdx oi, uintptr_t retaddr);
+void cpu_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
+ MemOpIdx oi, uintptr_t retaddr);
+
#if defined(CONFIG_USER_ONLY)
extern __thread uintptr_t helper_retaddr;
diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index 9f398b9afe..83e38487cf 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -33,7 +33,6 @@
#include "qemu/queue.h"
#include "tcg/tcg-mo.h"
#include "tcg-target.h"
-#include "qemu/int128.h"
#include "tcg/tcg-cond.h"
/* XXX: make safe guess about sizes */
@@ -1312,92 +1311,6 @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
#endif
#endif /* CONFIG_SOFTMMU */
-uint32_t cpu_atomic_cmpxchgb_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-uint32_t cpu_atomic_cmpxchgw_le_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-uint32_t cpu_atomic_cmpxchgl_le_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-uint64_t cpu_atomic_cmpxchgq_le_mmu(CPUArchState *env, target_ulong addr,
- uint64_t cmpv, uint64_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-uint32_t cpu_atomic_cmpxchgw_be_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-uint32_t cpu_atomic_cmpxchgl_be_mmu(CPUArchState *env, target_ulong addr,
- uint32_t cmpv, uint32_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-uint64_t cpu_atomic_cmpxchgq_be_mmu(CPUArchState *env, target_ulong addr,
- uint64_t cmpv, uint64_t newv,
- MemOpIdx oi, uintptr_t retaddr);
-
-#define GEN_ATOMIC_HELPER(NAME, TYPE, SUFFIX) \
-TYPE cpu_atomic_ ## NAME ## SUFFIX ## _mmu \
- (CPUArchState *env, target_ulong addr, TYPE val, \
- MemOpIdx oi, uintptr_t retaddr);
-
-#ifdef CONFIG_ATOMIC64
-#define GEN_ATOMIC_HELPER_ALL(NAME) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_be) \
- GEN_ATOMIC_HELPER(NAME, uint64_t, q_le) \
- GEN_ATOMIC_HELPER(NAME, uint64_t, q_be)
-#else
-#define GEN_ATOMIC_HELPER_ALL(NAME) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, b) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, w_be) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_le) \
- GEN_ATOMIC_HELPER(NAME, uint32_t, l_be)
-#endif
-
-GEN_ATOMIC_HELPER_ALL(fetch_add)
-GEN_ATOMIC_HELPER_ALL(fetch_sub)
-GEN_ATOMIC_HELPER_ALL(fetch_and)
-GEN_ATOMIC_HELPER_ALL(fetch_or)
-GEN_ATOMIC_HELPER_ALL(fetch_xor)
-GEN_ATOMIC_HELPER_ALL(fetch_smin)
-GEN_ATOMIC_HELPER_ALL(fetch_umin)
-GEN_ATOMIC_HELPER_ALL(fetch_smax)
-GEN_ATOMIC_HELPER_ALL(fetch_umax)
-
-GEN_ATOMIC_HELPER_ALL(add_fetch)
-GEN_ATOMIC_HELPER_ALL(sub_fetch)
-GEN_ATOMIC_HELPER_ALL(and_fetch)
-GEN_ATOMIC_HELPER_ALL(or_fetch)
-GEN_ATOMIC_HELPER_ALL(xor_fetch)
-GEN_ATOMIC_HELPER_ALL(smin_fetch)
-GEN_ATOMIC_HELPER_ALL(umin_fetch)
-GEN_ATOMIC_HELPER_ALL(smax_fetch)
-GEN_ATOMIC_HELPER_ALL(umax_fetch)
-
-GEN_ATOMIC_HELPER_ALL(xchg)
-
-#undef GEN_ATOMIC_HELPER_ALL
-#undef GEN_ATOMIC_HELPER
-
-Int128 cpu_atomic_cmpxchgo_le_mmu(CPUArchState *env, target_ulong addr,
- Int128 cmpv, Int128 newv,
- MemOpIdx oi, uintptr_t retaddr);
-Int128 cpu_atomic_cmpxchgo_be_mmu(CPUArchState *env, target_ulong addr,
- Int128 cmpv, Int128 newv,
- MemOpIdx oi, uintptr_t retaddr);
-
-Int128 cpu_atomic_ldo_le_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-Int128 cpu_atomic_ldo_be_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-void cpu_atomic_sto_le_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr);
-void cpu_atomic_sto_be_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr);
-
#ifdef CONFIG_DEBUG_TCG
void tcg_assert_listed_vecop(TCGOpcode);
#else
diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
index 4cafd3c11a..b110c57956 100644
--- a/target/arm/helper-a64.c
+++ b/target/arm/helper-a64.c
@@ -32,7 +32,6 @@
#include "exec/cpu_ldst.h"
#include "qemu/int128.h"
#include "qemu/atomic128.h"
-#include "tcg/tcg.h"
#include "fpu/softfloat.h"
#include <zlib.h> /* For crc32 */
diff --git a/target/m68k/op_helper.c b/target/m68k/op_helper.c
index c1bf73b6f9..cfbc987ba6 100644
--- a/target/m68k/op_helper.c
+++ b/target/m68k/op_helper.c
@@ -22,7 +22,6 @@
#include "exec/exec-all.h"
#include "exec/cpu_ldst.h"
#include "semihosting/semihost.h"
-#include "tcg/tcg.h"
#if !defined(CONFIG_USER_ONLY)
diff --git a/target/ppc/mem_helper.c b/target/ppc/mem_helper.c
index e2282baa8d..39945d9ea5 100644
--- a/target/ppc/mem_helper.c
+++ b/target/ppc/mem_helper.c
@@ -25,7 +25,6 @@
#include "exec/helper-proto.h"
#include "helper_regs.h"
#include "exec/cpu_ldst.h"
-#include "tcg/tcg.h"
#include "internal.h"
#include "qemu/atomic128.h"
diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c
index e64d1bc725..251d4acf55 100644
--- a/target/s390x/tcg/mem_helper.c
+++ b/target/s390x/tcg/mem_helper.c
@@ -27,7 +27,6 @@
#include "exec/cpu_ldst.h"
#include "qemu/int128.h"
#include "qemu/atomic128.h"
-#include "tcg/tcg.h"
#include "trace.h"
#if !defined(CONFIG_USER_ONLY)
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 09/15] target/mips: Use cpu_*_data_ra for msa load/store
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (7 preceding siblings ...)
2021-10-13 18:22 ` [PULL 08/15] accel/tcg: Move cpu_atomic decls to exec/cpu_ldst.h Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 10/15] target/mips: Use 8-byte memory ops " Richard Henderson
` (6 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé
We should not have been using the helper_ret_* set of
functions, as they are supposed to be private to tcg.
Nor should we have been using the plain cpu_*_data set
of functions, as they do not handle unwinding properly.
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/mips/tcg/msa_helper.c | 420 +++++++++++------------------------
1 file changed, 135 insertions(+), 285 deletions(-)
diff --git a/target/mips/tcg/msa_helper.c b/target/mips/tcg/msa_helper.c
index 167d9a591c..a8880ce81c 100644
--- a/target/mips/tcg/msa_helper.c
+++ b/target/mips/tcg/msa_helper.c
@@ -8222,79 +8222,42 @@ void helper_msa_ld_b(CPUMIPSState *env, uint32_t wd,
target_ulong addr)
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
- MEMOP_IDX(DF_BYTE)
-#if !defined(CONFIG_USER_ONLY)
+ uintptr_t ra = GETPC();
+
#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->b[0] = helper_ret_ldub_mmu(env, addr + (0 << DF_BYTE), oi, GETPC());
- pwd->b[1] = helper_ret_ldub_mmu(env, addr + (1 << DF_BYTE), oi, GETPC());
- pwd->b[2] = helper_ret_ldub_mmu(env, addr + (2 << DF_BYTE), oi, GETPC());
- pwd->b[3] = helper_ret_ldub_mmu(env, addr + (3 << DF_BYTE), oi, GETPC());
- pwd->b[4] = helper_ret_ldub_mmu(env, addr + (4 << DF_BYTE), oi, GETPC());
- pwd->b[5] = helper_ret_ldub_mmu(env, addr + (5 << DF_BYTE), oi, GETPC());
- pwd->b[6] = helper_ret_ldub_mmu(env, addr + (6 << DF_BYTE), oi, GETPC());
- pwd->b[7] = helper_ret_ldub_mmu(env, addr + (7 << DF_BYTE), oi, GETPC());
- pwd->b[8] = helper_ret_ldub_mmu(env, addr + (8 << DF_BYTE), oi, GETPC());
- pwd->b[9] = helper_ret_ldub_mmu(env, addr + (9 << DF_BYTE), oi, GETPC());
- pwd->b[10] = helper_ret_ldub_mmu(env, addr + (10 << DF_BYTE), oi, GETPC());
- pwd->b[11] = helper_ret_ldub_mmu(env, addr + (11 << DF_BYTE), oi, GETPC());
- pwd->b[12] = helper_ret_ldub_mmu(env, addr + (12 << DF_BYTE), oi, GETPC());
- pwd->b[13] = helper_ret_ldub_mmu(env, addr + (13 << DF_BYTE), oi, GETPC());
- pwd->b[14] = helper_ret_ldub_mmu(env, addr + (14 << DF_BYTE), oi, GETPC());
- pwd->b[15] = helper_ret_ldub_mmu(env, addr + (15 << DF_BYTE), oi, GETPC());
+ pwd->b[0] = cpu_ldub_data_ra(env, addr + (0 << DF_BYTE), ra);
+ pwd->b[1] = cpu_ldub_data_ra(env, addr + (1 << DF_BYTE), ra);
+ pwd->b[2] = cpu_ldub_data_ra(env, addr + (2 << DF_BYTE), ra);
+ pwd->b[3] = cpu_ldub_data_ra(env, addr + (3 << DF_BYTE), ra);
+ pwd->b[4] = cpu_ldub_data_ra(env, addr + (4 << DF_BYTE), ra);
+ pwd->b[5] = cpu_ldub_data_ra(env, addr + (5 << DF_BYTE), ra);
+ pwd->b[6] = cpu_ldub_data_ra(env, addr + (6 << DF_BYTE), ra);
+ pwd->b[7] = cpu_ldub_data_ra(env, addr + (7 << DF_BYTE), ra);
+ pwd->b[8] = cpu_ldub_data_ra(env, addr + (8 << DF_BYTE), ra);
+ pwd->b[9] = cpu_ldub_data_ra(env, addr + (9 << DF_BYTE), ra);
+ pwd->b[10] = cpu_ldub_data_ra(env, addr + (10 << DF_BYTE), ra);
+ pwd->b[11] = cpu_ldub_data_ra(env, addr + (11 << DF_BYTE), ra);
+ pwd->b[12] = cpu_ldub_data_ra(env, addr + (12 << DF_BYTE), ra);
+ pwd->b[13] = cpu_ldub_data_ra(env, addr + (13 << DF_BYTE), ra);
+ pwd->b[14] = cpu_ldub_data_ra(env, addr + (14 << DF_BYTE), ra);
+ pwd->b[15] = cpu_ldub_data_ra(env, addr + (15 << DF_BYTE), ra);
#else
- pwd->b[0] = helper_ret_ldub_mmu(env, addr + (7 << DF_BYTE), oi, GETPC());
- pwd->b[1] = helper_ret_ldub_mmu(env, addr + (6 << DF_BYTE), oi, GETPC());
- pwd->b[2] = helper_ret_ldub_mmu(env, addr + (5 << DF_BYTE), oi, GETPC());
- pwd->b[3] = helper_ret_ldub_mmu(env, addr + (4 << DF_BYTE), oi, GETPC());
- pwd->b[4] = helper_ret_ldub_mmu(env, addr + (3 << DF_BYTE), oi, GETPC());
- pwd->b[5] = helper_ret_ldub_mmu(env, addr + (2 << DF_BYTE), oi, GETPC());
- pwd->b[6] = helper_ret_ldub_mmu(env, addr + (1 << DF_BYTE), oi, GETPC());
- pwd->b[7] = helper_ret_ldub_mmu(env, addr + (0 << DF_BYTE), oi, GETPC());
- pwd->b[8] = helper_ret_ldub_mmu(env, addr + (15 << DF_BYTE), oi, GETPC());
- pwd->b[9] = helper_ret_ldub_mmu(env, addr + (14 << DF_BYTE), oi, GETPC());
- pwd->b[10] = helper_ret_ldub_mmu(env, addr + (13 << DF_BYTE), oi, GETPC());
- pwd->b[11] = helper_ret_ldub_mmu(env, addr + (12 << DF_BYTE), oi, GETPC());
- pwd->b[12] = helper_ret_ldub_mmu(env, addr + (11 << DF_BYTE), oi, GETPC());
- pwd->b[13] = helper_ret_ldub_mmu(env, addr + (10 << DF_BYTE), oi, GETPC());
- pwd->b[14] = helper_ret_ldub_mmu(env, addr + (9 << DF_BYTE), oi, GETPC());
- pwd->b[15] = helper_ret_ldub_mmu(env, addr + (8 << DF_BYTE), oi, GETPC());
-#endif
-#else
-#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->b[0] = cpu_ldub_data(env, addr + (0 << DF_BYTE));
- pwd->b[1] = cpu_ldub_data(env, addr + (1 << DF_BYTE));
- pwd->b[2] = cpu_ldub_data(env, addr + (2 << DF_BYTE));
- pwd->b[3] = cpu_ldub_data(env, addr + (3 << DF_BYTE));
- pwd->b[4] = cpu_ldub_data(env, addr + (4 << DF_BYTE));
- pwd->b[5] = cpu_ldub_data(env, addr + (5 << DF_BYTE));
- pwd->b[6] = cpu_ldub_data(env, addr + (6 << DF_BYTE));
- pwd->b[7] = cpu_ldub_data(env, addr + (7 << DF_BYTE));
- pwd->b[8] = cpu_ldub_data(env, addr + (8 << DF_BYTE));
- pwd->b[9] = cpu_ldub_data(env, addr + (9 << DF_BYTE));
- pwd->b[10] = cpu_ldub_data(env, addr + (10 << DF_BYTE));
- pwd->b[11] = cpu_ldub_data(env, addr + (11 << DF_BYTE));
- pwd->b[12] = cpu_ldub_data(env, addr + (12 << DF_BYTE));
- pwd->b[13] = cpu_ldub_data(env, addr + (13 << DF_BYTE));
- pwd->b[14] = cpu_ldub_data(env, addr + (14 << DF_BYTE));
- pwd->b[15] = cpu_ldub_data(env, addr + (15 << DF_BYTE));
-#else
- pwd->b[0] = cpu_ldub_data(env, addr + (7 << DF_BYTE));
- pwd->b[1] = cpu_ldub_data(env, addr + (6 << DF_BYTE));
- pwd->b[2] = cpu_ldub_data(env, addr + (5 << DF_BYTE));
- pwd->b[3] = cpu_ldub_data(env, addr + (4 << DF_BYTE));
- pwd->b[4] = cpu_ldub_data(env, addr + (3 << DF_BYTE));
- pwd->b[5] = cpu_ldub_data(env, addr + (2 << DF_BYTE));
- pwd->b[6] = cpu_ldub_data(env, addr + (1 << DF_BYTE));
- pwd->b[7] = cpu_ldub_data(env, addr + (0 << DF_BYTE));
- pwd->b[8] = cpu_ldub_data(env, addr + (15 << DF_BYTE));
- pwd->b[9] = cpu_ldub_data(env, addr + (14 << DF_BYTE));
- pwd->b[10] = cpu_ldub_data(env, addr + (13 << DF_BYTE));
- pwd->b[11] = cpu_ldub_data(env, addr + (12 << DF_BYTE));
- pwd->b[12] = cpu_ldub_data(env, addr + (11 << DF_BYTE));
- pwd->b[13] = cpu_ldub_data(env, addr + (10 << DF_BYTE));
- pwd->b[14] = cpu_ldub_data(env, addr + (9 << DF_BYTE));
- pwd->b[15] = cpu_ldub_data(env, addr + (8 << DF_BYTE));
-#endif
+ pwd->b[0] = cpu_ldub_data_ra(env, addr + (7 << DF_BYTE), ra);
+ pwd->b[1] = cpu_ldub_data_ra(env, addr + (6 << DF_BYTE), ra);
+ pwd->b[2] = cpu_ldub_data_ra(env, addr + (5 << DF_BYTE), ra);
+ pwd->b[3] = cpu_ldub_data_ra(env, addr + (4 << DF_BYTE), ra);
+ pwd->b[4] = cpu_ldub_data_ra(env, addr + (3 << DF_BYTE), ra);
+ pwd->b[5] = cpu_ldub_data_ra(env, addr + (2 << DF_BYTE), ra);
+ pwd->b[6] = cpu_ldub_data_ra(env, addr + (1 << DF_BYTE), ra);
+ pwd->b[7] = cpu_ldub_data_ra(env, addr + (0 << DF_BYTE), ra);
+ pwd->b[8] = cpu_ldub_data_ra(env, addr + (15 << DF_BYTE), ra);
+ pwd->b[9] = cpu_ldub_data_ra(env, addr + (14 << DF_BYTE), ra);
+ pwd->b[10] = cpu_ldub_data_ra(env, addr + (13 << DF_BYTE), ra);
+ pwd->b[11] = cpu_ldub_data_ra(env, addr + (12 << DF_BYTE), ra);
+ pwd->b[12] = cpu_ldub_data_ra(env, addr + (11 << DF_BYTE), ra);
+ pwd->b[13] = cpu_ldub_data_ra(env, addr + (10 << DF_BYTE), ra);
+ pwd->b[14] = cpu_ldub_data_ra(env, addr + (9 << DF_BYTE), ra);
+ pwd->b[15] = cpu_ldub_data_ra(env, addr + (8 << DF_BYTE), ra);
#endif
}
@@ -8302,47 +8265,26 @@ void helper_msa_ld_h(CPUMIPSState *env, uint32_t wd,
target_ulong addr)
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
- MEMOP_IDX(DF_HALF)
-#if !defined(CONFIG_USER_ONLY)
+ uintptr_t ra = GETPC();
+
#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->h[0] = helper_ret_lduw_mmu(env, addr + (0 << DF_HALF), oi, GETPC());
- pwd->h[1] = helper_ret_lduw_mmu(env, addr + (1 << DF_HALF), oi, GETPC());
- pwd->h[2] = helper_ret_lduw_mmu(env, addr + (2 << DF_HALF), oi, GETPC());
- pwd->h[3] = helper_ret_lduw_mmu(env, addr + (3 << DF_HALF), oi, GETPC());
- pwd->h[4] = helper_ret_lduw_mmu(env, addr + (4 << DF_HALF), oi, GETPC());
- pwd->h[5] = helper_ret_lduw_mmu(env, addr + (5 << DF_HALF), oi, GETPC());
- pwd->h[6] = helper_ret_lduw_mmu(env, addr + (6 << DF_HALF), oi, GETPC());
- pwd->h[7] = helper_ret_lduw_mmu(env, addr + (7 << DF_HALF), oi, GETPC());
+ pwd->h[0] = cpu_lduw_data_ra(env, addr + (0 << DF_HALF), ra);
+ pwd->h[1] = cpu_lduw_data_ra(env, addr + (1 << DF_HALF), ra);
+ pwd->h[2] = cpu_lduw_data_ra(env, addr + (2 << DF_HALF), ra);
+ pwd->h[3] = cpu_lduw_data_ra(env, addr + (3 << DF_HALF), ra);
+ pwd->h[4] = cpu_lduw_data_ra(env, addr + (4 << DF_HALF), ra);
+ pwd->h[5] = cpu_lduw_data_ra(env, addr + (5 << DF_HALF), ra);
+ pwd->h[6] = cpu_lduw_data_ra(env, addr + (6 << DF_HALF), ra);
+ pwd->h[7] = cpu_lduw_data_ra(env, addr + (7 << DF_HALF), ra);
#else
- pwd->h[0] = helper_ret_lduw_mmu(env, addr + (3 << DF_HALF), oi, GETPC());
- pwd->h[1] = helper_ret_lduw_mmu(env, addr + (2 << DF_HALF), oi, GETPC());
- pwd->h[2] = helper_ret_lduw_mmu(env, addr + (1 << DF_HALF), oi, GETPC());
- pwd->h[3] = helper_ret_lduw_mmu(env, addr + (0 << DF_HALF), oi, GETPC());
- pwd->h[4] = helper_ret_lduw_mmu(env, addr + (7 << DF_HALF), oi, GETPC());
- pwd->h[5] = helper_ret_lduw_mmu(env, addr + (6 << DF_HALF), oi, GETPC());
- pwd->h[6] = helper_ret_lduw_mmu(env, addr + (5 << DF_HALF), oi, GETPC());
- pwd->h[7] = helper_ret_lduw_mmu(env, addr + (4 << DF_HALF), oi, GETPC());
-#endif
-#else
-#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->h[0] = cpu_lduw_data(env, addr + (0 << DF_HALF));
- pwd->h[1] = cpu_lduw_data(env, addr + (1 << DF_HALF));
- pwd->h[2] = cpu_lduw_data(env, addr + (2 << DF_HALF));
- pwd->h[3] = cpu_lduw_data(env, addr + (3 << DF_HALF));
- pwd->h[4] = cpu_lduw_data(env, addr + (4 << DF_HALF));
- pwd->h[5] = cpu_lduw_data(env, addr + (5 << DF_HALF));
- pwd->h[6] = cpu_lduw_data(env, addr + (6 << DF_HALF));
- pwd->h[7] = cpu_lduw_data(env, addr + (7 << DF_HALF));
-#else
- pwd->h[0] = cpu_lduw_data(env, addr + (3 << DF_HALF));
- pwd->h[1] = cpu_lduw_data(env, addr + (2 << DF_HALF));
- pwd->h[2] = cpu_lduw_data(env, addr + (1 << DF_HALF));
- pwd->h[3] = cpu_lduw_data(env, addr + (0 << DF_HALF));
- pwd->h[4] = cpu_lduw_data(env, addr + (7 << DF_HALF));
- pwd->h[5] = cpu_lduw_data(env, addr + (6 << DF_HALF));
- pwd->h[6] = cpu_lduw_data(env, addr + (5 << DF_HALF));
- pwd->h[7] = cpu_lduw_data(env, addr + (4 << DF_HALF));
-#endif
+ pwd->h[0] = cpu_lduw_data_ra(env, addr + (3 << DF_HALF), ra);
+ pwd->h[1] = cpu_lduw_data_ra(env, addr + (2 << DF_HALF), ra);
+ pwd->h[2] = cpu_lduw_data_ra(env, addr + (1 << DF_HALF), ra);
+ pwd->h[3] = cpu_lduw_data_ra(env, addr + (0 << DF_HALF), ra);
+ pwd->h[4] = cpu_lduw_data_ra(env, addr + (7 << DF_HALF), ra);
+ pwd->h[5] = cpu_lduw_data_ra(env, addr + (6 << DF_HALF), ra);
+ pwd->h[6] = cpu_lduw_data_ra(env, addr + (5 << DF_HALF), ra);
+ pwd->h[7] = cpu_lduw_data_ra(env, addr + (4 << DF_HALF), ra);
#endif
}
@@ -8350,31 +8292,18 @@ void helper_msa_ld_w(CPUMIPSState *env, uint32_t wd,
target_ulong addr)
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
- MEMOP_IDX(DF_WORD)
-#if !defined(CONFIG_USER_ONLY)
+ uintptr_t ra = GETPC();
+
#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->w[0] = helper_ret_ldul_mmu(env, addr + (0 << DF_WORD), oi, GETPC());
- pwd->w[1] = helper_ret_ldul_mmu(env, addr + (1 << DF_WORD), oi, GETPC());
- pwd->w[2] = helper_ret_ldul_mmu(env, addr + (2 << DF_WORD), oi, GETPC());
- pwd->w[3] = helper_ret_ldul_mmu(env, addr + (3 << DF_WORD), oi, GETPC());
+ pwd->w[0] = cpu_ldl_data_ra(env, addr + (0 << DF_WORD), ra);
+ pwd->w[1] = cpu_ldl_data_ra(env, addr + (1 << DF_WORD), ra);
+ pwd->w[2] = cpu_ldl_data_ra(env, addr + (2 << DF_WORD), ra);
+ pwd->w[3] = cpu_ldl_data_ra(env, addr + (3 << DF_WORD), ra);
#else
- pwd->w[0] = helper_ret_ldul_mmu(env, addr + (1 << DF_WORD), oi, GETPC());
- pwd->w[1] = helper_ret_ldul_mmu(env, addr + (0 << DF_WORD), oi, GETPC());
- pwd->w[2] = helper_ret_ldul_mmu(env, addr + (3 << DF_WORD), oi, GETPC());
- pwd->w[3] = helper_ret_ldul_mmu(env, addr + (2 << DF_WORD), oi, GETPC());
-#endif
-#else
-#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->w[0] = cpu_ldl_data(env, addr + (0 << DF_WORD));
- pwd->w[1] = cpu_ldl_data(env, addr + (1 << DF_WORD));
- pwd->w[2] = cpu_ldl_data(env, addr + (2 << DF_WORD));
- pwd->w[3] = cpu_ldl_data(env, addr + (3 << DF_WORD));
-#else
- pwd->w[0] = cpu_ldl_data(env, addr + (1 << DF_WORD));
- pwd->w[1] = cpu_ldl_data(env, addr + (0 << DF_WORD));
- pwd->w[2] = cpu_ldl_data(env, addr + (3 << DF_WORD));
- pwd->w[3] = cpu_ldl_data(env, addr + (2 << DF_WORD));
-#endif
+ pwd->w[0] = cpu_ldl_data_ra(env, addr + (1 << DF_WORD), ra);
+ pwd->w[1] = cpu_ldl_data_ra(env, addr + (0 << DF_WORD), ra);
+ pwd->w[2] = cpu_ldl_data_ra(env, addr + (3 << DF_WORD), ra);
+ pwd->w[3] = cpu_ldl_data_ra(env, addr + (2 << DF_WORD), ra);
#endif
}
@@ -8382,14 +8311,10 @@ void helper_msa_ld_d(CPUMIPSState *env, uint32_t wd,
target_ulong addr)
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
- MEMOP_IDX(DF_DOUBLE)
-#if !defined(CONFIG_USER_ONLY)
- pwd->d[0] = helper_ret_ldq_mmu(env, addr + (0 << DF_DOUBLE), oi, GETPC());
- pwd->d[1] = helper_ret_ldq_mmu(env, addr + (1 << DF_DOUBLE), oi, GETPC());
-#else
- pwd->d[0] = cpu_ldq_data(env, addr + (0 << DF_DOUBLE));
- pwd->d[1] = cpu_ldq_data(env, addr + (1 << DF_DOUBLE));
-#endif
+ uintptr_t ra = GETPC();
+
+ pwd->d[0] = cpu_ldq_data_ra(env, addr + (0 << DF_DOUBLE), ra);
+ pwd->d[1] = cpu_ldq_data_ra(env, addr + (1 << DF_DOUBLE), ra);
}
#define MSA_PAGESPAN(x) \
@@ -8415,81 +8340,44 @@ void helper_msa_st_b(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
int mmu_idx = cpu_mmu_index(env, false);
+ uintptr_t ra = GETPC();
+
+ ensure_writable_pages(env, addr, mmu_idx, ra);
- MEMOP_IDX(DF_BYTE)
- ensure_writable_pages(env, addr, mmu_idx, GETPC());
-#if !defined(CONFIG_USER_ONLY)
#if !defined(HOST_WORDS_BIGENDIAN)
- helper_ret_stb_mmu(env, addr + (0 << DF_BYTE), pwd->b[0], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (1 << DF_BYTE), pwd->b[1], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (2 << DF_BYTE), pwd->b[2], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (3 << DF_BYTE), pwd->b[3], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (4 << DF_BYTE), pwd->b[4], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (5 << DF_BYTE), pwd->b[5], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (6 << DF_BYTE), pwd->b[6], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (7 << DF_BYTE), pwd->b[7], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (8 << DF_BYTE), pwd->b[8], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (9 << DF_BYTE), pwd->b[9], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (10 << DF_BYTE), pwd->b[10], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (11 << DF_BYTE), pwd->b[11], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (12 << DF_BYTE), pwd->b[12], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (13 << DF_BYTE), pwd->b[13], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (14 << DF_BYTE), pwd->b[14], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (15 << DF_BYTE), pwd->b[15], oi, GETPC());
+ cpu_stb_data_ra(env, addr + (0 << DF_BYTE), pwd->b[0], ra);
+ cpu_stb_data_ra(env, addr + (1 << DF_BYTE), pwd->b[1], ra);
+ cpu_stb_data_ra(env, addr + (2 << DF_BYTE), pwd->b[2], ra);
+ cpu_stb_data_ra(env, addr + (3 << DF_BYTE), pwd->b[3], ra);
+ cpu_stb_data_ra(env, addr + (4 << DF_BYTE), pwd->b[4], ra);
+ cpu_stb_data_ra(env, addr + (5 << DF_BYTE), pwd->b[5], ra);
+ cpu_stb_data_ra(env, addr + (6 << DF_BYTE), pwd->b[6], ra);
+ cpu_stb_data_ra(env, addr + (7 << DF_BYTE), pwd->b[7], ra);
+ cpu_stb_data_ra(env, addr + (8 << DF_BYTE), pwd->b[8], ra);
+ cpu_stb_data_ra(env, addr + (9 << DF_BYTE), pwd->b[9], ra);
+ cpu_stb_data_ra(env, addr + (10 << DF_BYTE), pwd->b[10], ra);
+ cpu_stb_data_ra(env, addr + (11 << DF_BYTE), pwd->b[11], ra);
+ cpu_stb_data_ra(env, addr + (12 << DF_BYTE), pwd->b[12], ra);
+ cpu_stb_data_ra(env, addr + (13 << DF_BYTE), pwd->b[13], ra);
+ cpu_stb_data_ra(env, addr + (14 << DF_BYTE), pwd->b[14], ra);
+ cpu_stb_data_ra(env, addr + (15 << DF_BYTE), pwd->b[15], ra);
#else
- helper_ret_stb_mmu(env, addr + (7 << DF_BYTE), pwd->b[0], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (6 << DF_BYTE), pwd->b[1], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (5 << DF_BYTE), pwd->b[2], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (4 << DF_BYTE), pwd->b[3], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (3 << DF_BYTE), pwd->b[4], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (2 << DF_BYTE), pwd->b[5], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (1 << DF_BYTE), pwd->b[6], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (0 << DF_BYTE), pwd->b[7], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (15 << DF_BYTE), pwd->b[8], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (14 << DF_BYTE), pwd->b[9], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (13 << DF_BYTE), pwd->b[10], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (12 << DF_BYTE), pwd->b[11], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (11 << DF_BYTE), pwd->b[12], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (10 << DF_BYTE), pwd->b[13], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (9 << DF_BYTE), pwd->b[14], oi, GETPC());
- helper_ret_stb_mmu(env, addr + (8 << DF_BYTE), pwd->b[15], oi, GETPC());
-#endif
-#else
-#if !defined(HOST_WORDS_BIGENDIAN)
- cpu_stb_data(env, addr + (0 << DF_BYTE), pwd->b[0]);
- cpu_stb_data(env, addr + (1 << DF_BYTE), pwd->b[1]);
- cpu_stb_data(env, addr + (2 << DF_BYTE), pwd->b[2]);
- cpu_stb_data(env, addr + (3 << DF_BYTE), pwd->b[3]);
- cpu_stb_data(env, addr + (4 << DF_BYTE), pwd->b[4]);
- cpu_stb_data(env, addr + (5 << DF_BYTE), pwd->b[5]);
- cpu_stb_data(env, addr + (6 << DF_BYTE), pwd->b[6]);
- cpu_stb_data(env, addr + (7 << DF_BYTE), pwd->b[7]);
- cpu_stb_data(env, addr + (8 << DF_BYTE), pwd->b[8]);
- cpu_stb_data(env, addr + (9 << DF_BYTE), pwd->b[9]);
- cpu_stb_data(env, addr + (10 << DF_BYTE), pwd->b[10]);
- cpu_stb_data(env, addr + (11 << DF_BYTE), pwd->b[11]);
- cpu_stb_data(env, addr + (12 << DF_BYTE), pwd->b[12]);
- cpu_stb_data(env, addr + (13 << DF_BYTE), pwd->b[13]);
- cpu_stb_data(env, addr + (14 << DF_BYTE), pwd->b[14]);
- cpu_stb_data(env, addr + (15 << DF_BYTE), pwd->b[15]);
-#else
- cpu_stb_data(env, addr + (7 << DF_BYTE), pwd->b[0]);
- cpu_stb_data(env, addr + (6 << DF_BYTE), pwd->b[1]);
- cpu_stb_data(env, addr + (5 << DF_BYTE), pwd->b[2]);
- cpu_stb_data(env, addr + (4 << DF_BYTE), pwd->b[3]);
- cpu_stb_data(env, addr + (3 << DF_BYTE), pwd->b[4]);
- cpu_stb_data(env, addr + (2 << DF_BYTE), pwd->b[5]);
- cpu_stb_data(env, addr + (1 << DF_BYTE), pwd->b[6]);
- cpu_stb_data(env, addr + (0 << DF_BYTE), pwd->b[7]);
- cpu_stb_data(env, addr + (15 << DF_BYTE), pwd->b[8]);
- cpu_stb_data(env, addr + (14 << DF_BYTE), pwd->b[9]);
- cpu_stb_data(env, addr + (13 << DF_BYTE), pwd->b[10]);
- cpu_stb_data(env, addr + (12 << DF_BYTE), pwd->b[11]);
- cpu_stb_data(env, addr + (11 << DF_BYTE), pwd->b[12]);
- cpu_stb_data(env, addr + (10 << DF_BYTE), pwd->b[13]);
- cpu_stb_data(env, addr + (9 << DF_BYTE), pwd->b[14]);
- cpu_stb_data(env, addr + (8 << DF_BYTE), pwd->b[15]);
-#endif
+ cpu_stb_data_ra(env, addr + (7 << DF_BYTE), pwd->b[0], ra);
+ cpu_stb_data_ra(env, addr + (6 << DF_BYTE), pwd->b[1], ra);
+ cpu_stb_data_ra(env, addr + (5 << DF_BYTE), pwd->b[2], ra);
+ cpu_stb_data_ra(env, addr + (4 << DF_BYTE), pwd->b[3], ra);
+ cpu_stb_data_ra(env, addr + (3 << DF_BYTE), pwd->b[4], ra);
+ cpu_stb_data_ra(env, addr + (2 << DF_BYTE), pwd->b[5], ra);
+ cpu_stb_data_ra(env, addr + (1 << DF_BYTE), pwd->b[6], ra);
+ cpu_stb_data_ra(env, addr + (0 << DF_BYTE), pwd->b[7], ra);
+ cpu_stb_data_ra(env, addr + (15 << DF_BYTE), pwd->b[8], ra);
+ cpu_stb_data_ra(env, addr + (14 << DF_BYTE), pwd->b[9], ra);
+ cpu_stb_data_ra(env, addr + (13 << DF_BYTE), pwd->b[10], ra);
+ cpu_stb_data_ra(env, addr + (12 << DF_BYTE), pwd->b[11], ra);
+ cpu_stb_data_ra(env, addr + (11 << DF_BYTE), pwd->b[12], ra);
+ cpu_stb_data_ra(env, addr + (10 << DF_BYTE), pwd->b[13], ra);
+ cpu_stb_data_ra(env, addr + (9 << DF_BYTE), pwd->b[14], ra);
+ cpu_stb_data_ra(env, addr + (8 << DF_BYTE), pwd->b[15], ra);
#endif
}
@@ -8498,49 +8386,28 @@ void helper_msa_st_h(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
int mmu_idx = cpu_mmu_index(env, false);
+ uintptr_t ra = GETPC();
+
+ ensure_writable_pages(env, addr, mmu_idx, ra);
- MEMOP_IDX(DF_HALF)
- ensure_writable_pages(env, addr, mmu_idx, GETPC());
-#if !defined(CONFIG_USER_ONLY)
#if !defined(HOST_WORDS_BIGENDIAN)
- helper_ret_stw_mmu(env, addr + (0 << DF_HALF), pwd->h[0], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (1 << DF_HALF), pwd->h[1], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (2 << DF_HALF), pwd->h[2], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (3 << DF_HALF), pwd->h[3], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (4 << DF_HALF), pwd->h[4], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (5 << DF_HALF), pwd->h[5], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (6 << DF_HALF), pwd->h[6], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (7 << DF_HALF), pwd->h[7], oi, GETPC());
+ cpu_stw_data_ra(env, addr + (0 << DF_HALF), pwd->h[0], ra);
+ cpu_stw_data_ra(env, addr + (1 << DF_HALF), pwd->h[1], ra);
+ cpu_stw_data_ra(env, addr + (2 << DF_HALF), pwd->h[2], ra);
+ cpu_stw_data_ra(env, addr + (3 << DF_HALF), pwd->h[3], ra);
+ cpu_stw_data_ra(env, addr + (4 << DF_HALF), pwd->h[4], ra);
+ cpu_stw_data_ra(env, addr + (5 << DF_HALF), pwd->h[5], ra);
+ cpu_stw_data_ra(env, addr + (6 << DF_HALF), pwd->h[6], ra);
+ cpu_stw_data_ra(env, addr + (7 << DF_HALF), pwd->h[7], ra);
#else
- helper_ret_stw_mmu(env, addr + (3 << DF_HALF), pwd->h[0], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (2 << DF_HALF), pwd->h[1], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (1 << DF_HALF), pwd->h[2], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (0 << DF_HALF), pwd->h[3], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (7 << DF_HALF), pwd->h[4], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (6 << DF_HALF), pwd->h[5], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (5 << DF_HALF), pwd->h[6], oi, GETPC());
- helper_ret_stw_mmu(env, addr + (4 << DF_HALF), pwd->h[7], oi, GETPC());
-#endif
-#else
-#if !defined(HOST_WORDS_BIGENDIAN)
- cpu_stw_data(env, addr + (0 << DF_HALF), pwd->h[0]);
- cpu_stw_data(env, addr + (1 << DF_HALF), pwd->h[1]);
- cpu_stw_data(env, addr + (2 << DF_HALF), pwd->h[2]);
- cpu_stw_data(env, addr + (3 << DF_HALF), pwd->h[3]);
- cpu_stw_data(env, addr + (4 << DF_HALF), pwd->h[4]);
- cpu_stw_data(env, addr + (5 << DF_HALF), pwd->h[5]);
- cpu_stw_data(env, addr + (6 << DF_HALF), pwd->h[6]);
- cpu_stw_data(env, addr + (7 << DF_HALF), pwd->h[7]);
-#else
- cpu_stw_data(env, addr + (3 << DF_HALF), pwd->h[0]);
- cpu_stw_data(env, addr + (2 << DF_HALF), pwd->h[1]);
- cpu_stw_data(env, addr + (1 << DF_HALF), pwd->h[2]);
- cpu_stw_data(env, addr + (0 << DF_HALF), pwd->h[3]);
- cpu_stw_data(env, addr + (7 << DF_HALF), pwd->h[4]);
- cpu_stw_data(env, addr + (6 << DF_HALF), pwd->h[5]);
- cpu_stw_data(env, addr + (5 << DF_HALF), pwd->h[6]);
- cpu_stw_data(env, addr + (4 << DF_HALF), pwd->h[7]);
-#endif
+ cpu_stw_data_ra(env, addr + (3 << DF_HALF), pwd->h[0], ra);
+ cpu_stw_data_ra(env, addr + (2 << DF_HALF), pwd->h[1], ra);
+ cpu_stw_data_ra(env, addr + (1 << DF_HALF), pwd->h[2], ra);
+ cpu_stw_data_ra(env, addr + (0 << DF_HALF), pwd->h[3], ra);
+ cpu_stw_data_ra(env, addr + (7 << DF_HALF), pwd->h[4], ra);
+ cpu_stw_data_ra(env, addr + (6 << DF_HALF), pwd->h[5], ra);
+ cpu_stw_data_ra(env, addr + (5 << DF_HALF), pwd->h[6], ra);
+ cpu_stw_data_ra(env, addr + (4 << DF_HALF), pwd->h[7], ra);
#endif
}
@@ -8549,33 +8416,20 @@ void helper_msa_st_w(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
int mmu_idx = cpu_mmu_index(env, false);
+ uintptr_t ra = GETPC();
+
+ ensure_writable_pages(env, addr, mmu_idx, ra);
- MEMOP_IDX(DF_WORD)
- ensure_writable_pages(env, addr, mmu_idx, GETPC());
-#if !defined(CONFIG_USER_ONLY)
#if !defined(HOST_WORDS_BIGENDIAN)
- helper_ret_stl_mmu(env, addr + (0 << DF_WORD), pwd->w[0], oi, GETPC());
- helper_ret_stl_mmu(env, addr + (1 << DF_WORD), pwd->w[1], oi, GETPC());
- helper_ret_stl_mmu(env, addr + (2 << DF_WORD), pwd->w[2], oi, GETPC());
- helper_ret_stl_mmu(env, addr + (3 << DF_WORD), pwd->w[3], oi, GETPC());
+ cpu_stl_data_ra(env, addr + (0 << DF_WORD), pwd->w[0], ra);
+ cpu_stl_data_ra(env, addr + (1 << DF_WORD), pwd->w[1], ra);
+ cpu_stl_data_ra(env, addr + (2 << DF_WORD), pwd->w[2], ra);
+ cpu_stl_data_ra(env, addr + (3 << DF_WORD), pwd->w[3], ra);
#else
- helper_ret_stl_mmu(env, addr + (1 << DF_WORD), pwd->w[0], oi, GETPC());
- helper_ret_stl_mmu(env, addr + (0 << DF_WORD), pwd->w[1], oi, GETPC());
- helper_ret_stl_mmu(env, addr + (3 << DF_WORD), pwd->w[2], oi, GETPC());
- helper_ret_stl_mmu(env, addr + (2 << DF_WORD), pwd->w[3], oi, GETPC());
-#endif
-#else
-#if !defined(HOST_WORDS_BIGENDIAN)
- cpu_stl_data(env, addr + (0 << DF_WORD), pwd->w[0]);
- cpu_stl_data(env, addr + (1 << DF_WORD), pwd->w[1]);
- cpu_stl_data(env, addr + (2 << DF_WORD), pwd->w[2]);
- cpu_stl_data(env, addr + (3 << DF_WORD), pwd->w[3]);
-#else
- cpu_stl_data(env, addr + (1 << DF_WORD), pwd->w[0]);
- cpu_stl_data(env, addr + (0 << DF_WORD), pwd->w[1]);
- cpu_stl_data(env, addr + (3 << DF_WORD), pwd->w[2]);
- cpu_stl_data(env, addr + (2 << DF_WORD), pwd->w[3]);
-#endif
+ cpu_stl_data_ra(env, addr + (1 << DF_WORD), pwd->w[0], ra);
+ cpu_stl_data_ra(env, addr + (0 << DF_WORD), pwd->w[1], ra);
+ cpu_stl_data_ra(env, addr + (3 << DF_WORD), pwd->w[2], ra);
+ cpu_stl_data_ra(env, addr + (2 << DF_WORD), pwd->w[3], ra);
#endif
}
@@ -8584,14 +8438,10 @@ void helper_msa_st_d(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
int mmu_idx = cpu_mmu_index(env, false);
+ uintptr_t ra = GETPC();
- MEMOP_IDX(DF_DOUBLE)
ensure_writable_pages(env, addr, mmu_idx, GETPC());
-#if !defined(CONFIG_USER_ONLY)
- helper_ret_stq_mmu(env, addr + (0 << DF_DOUBLE), pwd->d[0], oi, GETPC());
- helper_ret_stq_mmu(env, addr + (1 << DF_DOUBLE), pwd->d[1], oi, GETPC());
-#else
- cpu_stq_data(env, addr + (0 << DF_DOUBLE), pwd->d[0]);
- cpu_stq_data(env, addr + (1 << DF_DOUBLE), pwd->d[1]);
-#endif
+
+ cpu_stq_data_ra(env, addr + (0 << DF_DOUBLE), pwd->d[0], ra);
+ cpu_stq_data_ra(env, addr + (1 << DF_DOUBLE), pwd->d[1], ra);
}
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 10/15] target/mips: Use 8-byte memory ops for msa load/store
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (8 preceding siblings ...)
2021-10-13 18:22 ` [PULL 09/15] target/mips: Use cpu_*_data_ra for msa load/store Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 11/15] target/s390x: Use cpu_*_mmu instead of helper_*_mmu Richard Henderson
` (5 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé
Rather than use 4-16 separate operations, use 2 operations
plus some byte reordering as necessary.
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/mips/tcg/msa_helper.c | 201 +++++++++++++----------------------
1 file changed, 71 insertions(+), 130 deletions(-)
diff --git a/target/mips/tcg/msa_helper.c b/target/mips/tcg/msa_helper.c
index a8880ce81c..e40c1b7057 100644
--- a/target/mips/tcg/msa_helper.c
+++ b/target/mips/tcg/msa_helper.c
@@ -8218,47 +8218,31 @@ void helper_msa_ffint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
#define MEMOP_IDX(DF)
#endif
+#ifdef TARGET_WORDS_BIGENDIAN
+static inline uint64_t bswap16x4(uint64_t x)
+{
+ uint64_t m = 0x00ff00ff00ff00ffull;
+ return ((x & m) << 8) | ((x >> 8) & m);
+}
+
+static inline uint64_t bswap32x2(uint64_t x)
+{
+ return ror64(bswap64(x), 32);
+}
+#endif
+
void helper_msa_ld_b(CPUMIPSState *env, uint32_t wd,
target_ulong addr)
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
uintptr_t ra = GETPC();
+ uint64_t d0, d1;
-#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->b[0] = cpu_ldub_data_ra(env, addr + (0 << DF_BYTE), ra);
- pwd->b[1] = cpu_ldub_data_ra(env, addr + (1 << DF_BYTE), ra);
- pwd->b[2] = cpu_ldub_data_ra(env, addr + (2 << DF_BYTE), ra);
- pwd->b[3] = cpu_ldub_data_ra(env, addr + (3 << DF_BYTE), ra);
- pwd->b[4] = cpu_ldub_data_ra(env, addr + (4 << DF_BYTE), ra);
- pwd->b[5] = cpu_ldub_data_ra(env, addr + (5 << DF_BYTE), ra);
- pwd->b[6] = cpu_ldub_data_ra(env, addr + (6 << DF_BYTE), ra);
- pwd->b[7] = cpu_ldub_data_ra(env, addr + (7 << DF_BYTE), ra);
- pwd->b[8] = cpu_ldub_data_ra(env, addr + (8 << DF_BYTE), ra);
- pwd->b[9] = cpu_ldub_data_ra(env, addr + (9 << DF_BYTE), ra);
- pwd->b[10] = cpu_ldub_data_ra(env, addr + (10 << DF_BYTE), ra);
- pwd->b[11] = cpu_ldub_data_ra(env, addr + (11 << DF_BYTE), ra);
- pwd->b[12] = cpu_ldub_data_ra(env, addr + (12 << DF_BYTE), ra);
- pwd->b[13] = cpu_ldub_data_ra(env, addr + (13 << DF_BYTE), ra);
- pwd->b[14] = cpu_ldub_data_ra(env, addr + (14 << DF_BYTE), ra);
- pwd->b[15] = cpu_ldub_data_ra(env, addr + (15 << DF_BYTE), ra);
-#else
- pwd->b[0] = cpu_ldub_data_ra(env, addr + (7 << DF_BYTE), ra);
- pwd->b[1] = cpu_ldub_data_ra(env, addr + (6 << DF_BYTE), ra);
- pwd->b[2] = cpu_ldub_data_ra(env, addr + (5 << DF_BYTE), ra);
- pwd->b[3] = cpu_ldub_data_ra(env, addr + (4 << DF_BYTE), ra);
- pwd->b[4] = cpu_ldub_data_ra(env, addr + (3 << DF_BYTE), ra);
- pwd->b[5] = cpu_ldub_data_ra(env, addr + (2 << DF_BYTE), ra);
- pwd->b[6] = cpu_ldub_data_ra(env, addr + (1 << DF_BYTE), ra);
- pwd->b[7] = cpu_ldub_data_ra(env, addr + (0 << DF_BYTE), ra);
- pwd->b[8] = cpu_ldub_data_ra(env, addr + (15 << DF_BYTE), ra);
- pwd->b[9] = cpu_ldub_data_ra(env, addr + (14 << DF_BYTE), ra);
- pwd->b[10] = cpu_ldub_data_ra(env, addr + (13 << DF_BYTE), ra);
- pwd->b[11] = cpu_ldub_data_ra(env, addr + (12 << DF_BYTE), ra);
- pwd->b[12] = cpu_ldub_data_ra(env, addr + (11 << DF_BYTE), ra);
- pwd->b[13] = cpu_ldub_data_ra(env, addr + (10 << DF_BYTE), ra);
- pwd->b[14] = cpu_ldub_data_ra(env, addr + (9 << DF_BYTE), ra);
- pwd->b[15] = cpu_ldub_data_ra(env, addr + (8 << DF_BYTE), ra);
-#endif
+ /* Load 8 bytes at a time. Vector element ordering makes this LE. */
+ d0 = cpu_ldq_le_data_ra(env, addr + 0, ra);
+ d1 = cpu_ldq_le_data_ra(env, addr + 8, ra);
+ pwd->d[0] = d0;
+ pwd->d[1] = d1;
}
void helper_msa_ld_h(CPUMIPSState *env, uint32_t wd,
@@ -8266,26 +8250,20 @@ void helper_msa_ld_h(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
uintptr_t ra = GETPC();
+ uint64_t d0, d1;
-#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->h[0] = cpu_lduw_data_ra(env, addr + (0 << DF_HALF), ra);
- pwd->h[1] = cpu_lduw_data_ra(env, addr + (1 << DF_HALF), ra);
- pwd->h[2] = cpu_lduw_data_ra(env, addr + (2 << DF_HALF), ra);
- pwd->h[3] = cpu_lduw_data_ra(env, addr + (3 << DF_HALF), ra);
- pwd->h[4] = cpu_lduw_data_ra(env, addr + (4 << DF_HALF), ra);
- pwd->h[5] = cpu_lduw_data_ra(env, addr + (5 << DF_HALF), ra);
- pwd->h[6] = cpu_lduw_data_ra(env, addr + (6 << DF_HALF), ra);
- pwd->h[7] = cpu_lduw_data_ra(env, addr + (7 << DF_HALF), ra);
-#else
- pwd->h[0] = cpu_lduw_data_ra(env, addr + (3 << DF_HALF), ra);
- pwd->h[1] = cpu_lduw_data_ra(env, addr + (2 << DF_HALF), ra);
- pwd->h[2] = cpu_lduw_data_ra(env, addr + (1 << DF_HALF), ra);
- pwd->h[3] = cpu_lduw_data_ra(env, addr + (0 << DF_HALF), ra);
- pwd->h[4] = cpu_lduw_data_ra(env, addr + (7 << DF_HALF), ra);
- pwd->h[5] = cpu_lduw_data_ra(env, addr + (6 << DF_HALF), ra);
- pwd->h[6] = cpu_lduw_data_ra(env, addr + (5 << DF_HALF), ra);
- pwd->h[7] = cpu_lduw_data_ra(env, addr + (4 << DF_HALF), ra);
+ /*
+ * Load 8 bytes at a time. Use little-endian load, then for
+ * big-endian target, we must then swap the four halfwords.
+ */
+ d0 = cpu_ldq_le_data_ra(env, addr + 0, ra);
+ d1 = cpu_ldq_le_data_ra(env, addr + 8, ra);
+#ifdef TARGET_WORDS_BIGENDIAN
+ d0 = bswap16x4(d0);
+ d1 = bswap16x4(d1);
#endif
+ pwd->d[0] = d0;
+ pwd->d[1] = d1;
}
void helper_msa_ld_w(CPUMIPSState *env, uint32_t wd,
@@ -8293,18 +8271,20 @@ void helper_msa_ld_w(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
uintptr_t ra = GETPC();
+ uint64_t d0, d1;
-#if !defined(HOST_WORDS_BIGENDIAN)
- pwd->w[0] = cpu_ldl_data_ra(env, addr + (0 << DF_WORD), ra);
- pwd->w[1] = cpu_ldl_data_ra(env, addr + (1 << DF_WORD), ra);
- pwd->w[2] = cpu_ldl_data_ra(env, addr + (2 << DF_WORD), ra);
- pwd->w[3] = cpu_ldl_data_ra(env, addr + (3 << DF_WORD), ra);
-#else
- pwd->w[0] = cpu_ldl_data_ra(env, addr + (1 << DF_WORD), ra);
- pwd->w[1] = cpu_ldl_data_ra(env, addr + (0 << DF_WORD), ra);
- pwd->w[2] = cpu_ldl_data_ra(env, addr + (3 << DF_WORD), ra);
- pwd->w[3] = cpu_ldl_data_ra(env, addr + (2 << DF_WORD), ra);
+ /*
+ * Load 8 bytes at a time. Use little-endian load, then for
+ * big-endian target, we must then bswap the two words.
+ */
+ d0 = cpu_ldq_le_data_ra(env, addr + 0, ra);
+ d1 = cpu_ldq_le_data_ra(env, addr + 8, ra);
+#ifdef TARGET_WORDS_BIGENDIAN
+ d0 = bswap32x2(d0);
+ d1 = bswap32x2(d1);
#endif
+ pwd->d[0] = d0;
+ pwd->d[1] = d1;
}
void helper_msa_ld_d(CPUMIPSState *env, uint32_t wd,
@@ -8312,9 +8292,12 @@ void helper_msa_ld_d(CPUMIPSState *env, uint32_t wd,
{
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
uintptr_t ra = GETPC();
+ uint64_t d0, d1;
- pwd->d[0] = cpu_ldq_data_ra(env, addr + (0 << DF_DOUBLE), ra);
- pwd->d[1] = cpu_ldq_data_ra(env, addr + (1 << DF_DOUBLE), ra);
+ d0 = cpu_ldq_data_ra(env, addr + 0, ra);
+ d1 = cpu_ldq_data_ra(env, addr + 8, ra);
+ pwd->d[0] = d0;
+ pwd->d[1] = d1;
}
#define MSA_PAGESPAN(x) \
@@ -8344,41 +8327,9 @@ void helper_msa_st_b(CPUMIPSState *env, uint32_t wd,
ensure_writable_pages(env, addr, mmu_idx, ra);
-#if !defined(HOST_WORDS_BIGENDIAN)
- cpu_stb_data_ra(env, addr + (0 << DF_BYTE), pwd->b[0], ra);
- cpu_stb_data_ra(env, addr + (1 << DF_BYTE), pwd->b[1], ra);
- cpu_stb_data_ra(env, addr + (2 << DF_BYTE), pwd->b[2], ra);
- cpu_stb_data_ra(env, addr + (3 << DF_BYTE), pwd->b[3], ra);
- cpu_stb_data_ra(env, addr + (4 << DF_BYTE), pwd->b[4], ra);
- cpu_stb_data_ra(env, addr + (5 << DF_BYTE), pwd->b[5], ra);
- cpu_stb_data_ra(env, addr + (6 << DF_BYTE), pwd->b[6], ra);
- cpu_stb_data_ra(env, addr + (7 << DF_BYTE), pwd->b[7], ra);
- cpu_stb_data_ra(env, addr + (8 << DF_BYTE), pwd->b[8], ra);
- cpu_stb_data_ra(env, addr + (9 << DF_BYTE), pwd->b[9], ra);
- cpu_stb_data_ra(env, addr + (10 << DF_BYTE), pwd->b[10], ra);
- cpu_stb_data_ra(env, addr + (11 << DF_BYTE), pwd->b[11], ra);
- cpu_stb_data_ra(env, addr + (12 << DF_BYTE), pwd->b[12], ra);
- cpu_stb_data_ra(env, addr + (13 << DF_BYTE), pwd->b[13], ra);
- cpu_stb_data_ra(env, addr + (14 << DF_BYTE), pwd->b[14], ra);
- cpu_stb_data_ra(env, addr + (15 << DF_BYTE), pwd->b[15], ra);
-#else
- cpu_stb_data_ra(env, addr + (7 << DF_BYTE), pwd->b[0], ra);
- cpu_stb_data_ra(env, addr + (6 << DF_BYTE), pwd->b[1], ra);
- cpu_stb_data_ra(env, addr + (5 << DF_BYTE), pwd->b[2], ra);
- cpu_stb_data_ra(env, addr + (4 << DF_BYTE), pwd->b[3], ra);
- cpu_stb_data_ra(env, addr + (3 << DF_BYTE), pwd->b[4], ra);
- cpu_stb_data_ra(env, addr + (2 << DF_BYTE), pwd->b[5], ra);
- cpu_stb_data_ra(env, addr + (1 << DF_BYTE), pwd->b[6], ra);
- cpu_stb_data_ra(env, addr + (0 << DF_BYTE), pwd->b[7], ra);
- cpu_stb_data_ra(env, addr + (15 << DF_BYTE), pwd->b[8], ra);
- cpu_stb_data_ra(env, addr + (14 << DF_BYTE), pwd->b[9], ra);
- cpu_stb_data_ra(env, addr + (13 << DF_BYTE), pwd->b[10], ra);
- cpu_stb_data_ra(env, addr + (12 << DF_BYTE), pwd->b[11], ra);
- cpu_stb_data_ra(env, addr + (11 << DF_BYTE), pwd->b[12], ra);
- cpu_stb_data_ra(env, addr + (10 << DF_BYTE), pwd->b[13], ra);
- cpu_stb_data_ra(env, addr + (9 << DF_BYTE), pwd->b[14], ra);
- cpu_stb_data_ra(env, addr + (8 << DF_BYTE), pwd->b[15], ra);
-#endif
+ /* Store 8 bytes at a time. Vector element ordering makes this LE. */
+ cpu_stq_le_data_ra(env, addr + 0, pwd->d[0], ra);
+ cpu_stq_le_data_ra(env, addr + 0, pwd->d[1], ra);
}
void helper_msa_st_h(CPUMIPSState *env, uint32_t wd,
@@ -8387,28 +8338,19 @@ void helper_msa_st_h(CPUMIPSState *env, uint32_t wd,
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
int mmu_idx = cpu_mmu_index(env, false);
uintptr_t ra = GETPC();
+ uint64_t d0, d1;
ensure_writable_pages(env, addr, mmu_idx, ra);
-#if !defined(HOST_WORDS_BIGENDIAN)
- cpu_stw_data_ra(env, addr + (0 << DF_HALF), pwd->h[0], ra);
- cpu_stw_data_ra(env, addr + (1 << DF_HALF), pwd->h[1], ra);
- cpu_stw_data_ra(env, addr + (2 << DF_HALF), pwd->h[2], ra);
- cpu_stw_data_ra(env, addr + (3 << DF_HALF), pwd->h[3], ra);
- cpu_stw_data_ra(env, addr + (4 << DF_HALF), pwd->h[4], ra);
- cpu_stw_data_ra(env, addr + (5 << DF_HALF), pwd->h[5], ra);
- cpu_stw_data_ra(env, addr + (6 << DF_HALF), pwd->h[6], ra);
- cpu_stw_data_ra(env, addr + (7 << DF_HALF), pwd->h[7], ra);
-#else
- cpu_stw_data_ra(env, addr + (3 << DF_HALF), pwd->h[0], ra);
- cpu_stw_data_ra(env, addr + (2 << DF_HALF), pwd->h[1], ra);
- cpu_stw_data_ra(env, addr + (1 << DF_HALF), pwd->h[2], ra);
- cpu_stw_data_ra(env, addr + (0 << DF_HALF), pwd->h[3], ra);
- cpu_stw_data_ra(env, addr + (7 << DF_HALF), pwd->h[4], ra);
- cpu_stw_data_ra(env, addr + (6 << DF_HALF), pwd->h[5], ra);
- cpu_stw_data_ra(env, addr + (5 << DF_HALF), pwd->h[6], ra);
- cpu_stw_data_ra(env, addr + (4 << DF_HALF), pwd->h[7], ra);
+ /* Store 8 bytes at a time. See helper_msa_ld_h. */
+ d0 = pwd->d[0];
+ d1 = pwd->d[1];
+#ifdef TARGET_WORDS_BIGENDIAN
+ d0 = bswap16x4(d0);
+ d1 = bswap16x4(d1);
#endif
+ cpu_stq_le_data_ra(env, addr + 0, d0, ra);
+ cpu_stq_le_data_ra(env, addr + 8, d1, ra);
}
void helper_msa_st_w(CPUMIPSState *env, uint32_t wd,
@@ -8417,20 +8359,19 @@ void helper_msa_st_w(CPUMIPSState *env, uint32_t wd,
wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
int mmu_idx = cpu_mmu_index(env, false);
uintptr_t ra = GETPC();
+ uint64_t d0, d1;
ensure_writable_pages(env, addr, mmu_idx, ra);
-#if !defined(HOST_WORDS_BIGENDIAN)
- cpu_stl_data_ra(env, addr + (0 << DF_WORD), pwd->w[0], ra);
- cpu_stl_data_ra(env, addr + (1 << DF_WORD), pwd->w[1], ra);
- cpu_stl_data_ra(env, addr + (2 << DF_WORD), pwd->w[2], ra);
- cpu_stl_data_ra(env, addr + (3 << DF_WORD), pwd->w[3], ra);
-#else
- cpu_stl_data_ra(env, addr + (1 << DF_WORD), pwd->w[0], ra);
- cpu_stl_data_ra(env, addr + (0 << DF_WORD), pwd->w[1], ra);
- cpu_stl_data_ra(env, addr + (3 << DF_WORD), pwd->w[2], ra);
- cpu_stl_data_ra(env, addr + (2 << DF_WORD), pwd->w[3], ra);
+ /* Store 8 bytes at a time. See helper_msa_ld_w. */
+ d0 = pwd->d[0];
+ d1 = pwd->d[1];
+#ifdef TARGET_WORDS_BIGENDIAN
+ d0 = bswap32x2(d0);
+ d1 = bswap32x2(d1);
#endif
+ cpu_stq_le_data_ra(env, addr + 0, d0, ra);
+ cpu_stq_le_data_ra(env, addr + 8, d1, ra);
}
void helper_msa_st_d(CPUMIPSState *env, uint32_t wd,
@@ -8442,6 +8383,6 @@ void helper_msa_st_d(CPUMIPSState *env, uint32_t wd,
ensure_writable_pages(env, addr, mmu_idx, GETPC());
- cpu_stq_data_ra(env, addr + (0 << DF_DOUBLE), pwd->d[0], ra);
- cpu_stq_data_ra(env, addr + (1 << DF_DOUBLE), pwd->d[1], ra);
+ cpu_stq_data_ra(env, addr + 0, pwd->d[0], ra);
+ cpu_stq_data_ra(env, addr + 8, pwd->d[1], ra);
}
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 11/15] target/s390x: Use cpu_*_mmu instead of helper_*_mmu
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (9 preceding siblings ...)
2021-10-13 18:22 ` [PULL 10/15] target/mips: Use 8-byte memory ops " Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 12/15] target/sparc: " Richard Henderson
` (4 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé, David Hildenbrand
The helper_*_mmu functions were the only thing available
when this code was written. This could have been adjusted
when we added cpu_*_mmuidx_ra, but now we can most easily
use the newest set of interfaces.
Reviewed-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/s390x/tcg/mem_helper.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/target/s390x/tcg/mem_helper.c b/target/s390x/tcg/mem_helper.c
index 251d4acf55..17e3f83641 100644
--- a/target/s390x/tcg/mem_helper.c
+++ b/target/s390x/tcg/mem_helper.c
@@ -249,13 +249,13 @@ static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr,
* page. This is especially relevant to speed up TLB_NOTDIRTY.
*/
g_assert(size > 0);
- helper_ret_stb_mmu(env, vaddr, byte, oi, ra);
+ cpu_stb_mmu(env, vaddr, byte, oi, ra);
haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
if (likely(haddr)) {
memset(haddr + 1, byte, size - 1);
} else {
for (i = 1; i < size; i++) {
- helper_ret_stb_mmu(env, vaddr + i, byte, oi, ra);
+ cpu_stb_mmu(env, vaddr + i, byte, oi, ra);
}
}
}
@@ -291,7 +291,7 @@ static uint8_t do_access_get_byte(CPUS390XState *env, vaddr vaddr, char **haddr,
* Do a single access and test if we can then get access to the
* page. This is especially relevant to speed up TLB_NOTDIRTY.
*/
- byte = helper_ret_ldub_mmu(env, vaddr + offset, oi, ra);
+ byte = cpu_ldb_mmu(env, vaddr + offset, oi, ra);
*haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_LOAD, mmu_idx);
return byte;
#endif
@@ -325,7 +325,7 @@ static void do_access_set_byte(CPUS390XState *env, vaddr vaddr, char **haddr,
* Do a single access and test if we can then get access to the
* page. This is especially relevant to speed up TLB_NOTDIRTY.
*/
- helper_ret_stb_mmu(env, vaddr + offset, byte, oi, ra);
+ cpu_stb_mmu(env, vaddr + offset, byte, oi, ra);
*haddr = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
#endif
}
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 12/15] target/sparc: Use cpu_*_mmu instead of helper_*_mmu
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (10 preceding siblings ...)
2021-10-13 18:22 ` [PULL 11/15] target/s390x: Use cpu_*_mmu instead of helper_*_mmu Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 13/15] target/arm: " Richard Henderson
` (3 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Mark Cave-Ayland, Philippe Mathieu-Daudé
The helper_*_mmu functions were the only thing available
when this code was written. This could have been adjusted
when we added cpu_*_mmuidx_ra, but now we can most easily
use the newest set of interfaces.
Reviewed-by: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk>
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/sparc/ldst_helper.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/target/sparc/ldst_helper.c b/target/sparc/ldst_helper.c
index abe2889d27..bbf3601cb1 100644
--- a/target/sparc/ldst_helper.c
+++ b/target/sparc/ldst_helper.c
@@ -1333,27 +1333,27 @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr,
oi = make_memop_idx(memop, idx);
switch (size) {
case 1:
- ret = helper_ret_ldub_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldb_mmu(env, addr, oi, GETPC());
break;
case 2:
if (asi & 8) {
- ret = helper_le_lduw_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldw_le_mmu(env, addr, oi, GETPC());
} else {
- ret = helper_be_lduw_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldw_be_mmu(env, addr, oi, GETPC());
}
break;
case 4:
if (asi & 8) {
- ret = helper_le_ldul_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldl_le_mmu(env, addr, oi, GETPC());
} else {
- ret = helper_be_ldul_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldl_be_mmu(env, addr, oi, GETPC());
}
break;
case 8:
if (asi & 8) {
- ret = helper_le_ldq_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldq_le_mmu(env, addr, oi, GETPC());
} else {
- ret = helper_be_ldq_mmu(env, addr, oi, GETPC());
+ ret = cpu_ldq_be_mmu(env, addr, oi, GETPC());
}
break;
default:
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 13/15] target/arm: Use cpu_*_mmu instead of helper_*_mmu
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (11 preceding siblings ...)
2021-10-13 18:22 ` [PULL 12/15] target/sparc: " Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 14/15] tcg: Move helper_*_mmu decls to tcg/tcg-ldst.h Richard Henderson
` (2 subsequent siblings)
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: qemu-arm, Philippe Mathieu-Daudé
The helper_*_mmu functions were the only thing available
when this code was written. This could have been adjusted
when we added cpu_*_mmuidx_ra, but now we can most easily
use the newest set of interfaces.
Cc: qemu-arm@nongnu.org
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
target/arm/helper-a64.c | 52 +++++++----------------------------------
target/arm/m_helper.c | 6 ++---
2 files changed, 11 insertions(+), 47 deletions(-)
diff --git a/target/arm/helper-a64.c b/target/arm/helper-a64.c
index b110c57956..5ae2ecb0f3 100644
--- a/target/arm/helper-a64.c
+++ b/target/arm/helper-a64.c
@@ -512,37 +512,19 @@ uint64_t HELPER(paired_cmpxchg64_le)(CPUARMState *env, uint64_t addr,
uintptr_t ra = GETPC();
uint64_t o0, o1;
bool success;
-
-#ifdef CONFIG_USER_ONLY
- /* ??? Enforce alignment. */
- uint64_t *haddr = g2h(env_cpu(env), addr);
-
- set_helper_retaddr(ra);
- o0 = ldq_le_p(haddr + 0);
- o1 = ldq_le_p(haddr + 1);
- oldv = int128_make128(o0, o1);
-
- success = int128_eq(oldv, cmpv);
- if (success) {
- stq_le_p(haddr + 0, int128_getlo(newv));
- stq_le_p(haddr + 1, int128_gethi(newv));
- }
- clear_helper_retaddr();
-#else
int mem_idx = cpu_mmu_index(env, false);
MemOpIdx oi0 = make_memop_idx(MO_LEQ | MO_ALIGN_16, mem_idx);
MemOpIdx oi1 = make_memop_idx(MO_LEQ, mem_idx);
- o0 = helper_le_ldq_mmu(env, addr + 0, oi0, ra);
- o1 = helper_le_ldq_mmu(env, addr + 8, oi1, ra);
+ o0 = cpu_ldq_le_mmu(env, addr + 0, oi0, ra);
+ o1 = cpu_ldq_le_mmu(env, addr + 8, oi1, ra);
oldv = int128_make128(o0, o1);
success = int128_eq(oldv, cmpv);
if (success) {
- helper_le_stq_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
- helper_le_stq_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
+ cpu_stq_le_mmu(env, addr + 0, int128_getlo(newv), oi1, ra);
+ cpu_stq_le_mmu(env, addr + 8, int128_gethi(newv), oi1, ra);
}
-#endif
return !success;
}
@@ -582,37 +564,19 @@ uint64_t HELPER(paired_cmpxchg64_be)(CPUARMState *env, uint64_t addr,
uintptr_t ra = GETPC();
uint64_t o0, o1;
bool success;
-
-#ifdef CONFIG_USER_ONLY
- /* ??? Enforce alignment. */
- uint64_t *haddr = g2h(env_cpu(env), addr);
-
- set_helper_retaddr(ra);
- o1 = ldq_be_p(haddr + 0);
- o0 = ldq_be_p(haddr + 1);
- oldv = int128_make128(o0, o1);
-
- success = int128_eq(oldv, cmpv);
- if (success) {
- stq_be_p(haddr + 0, int128_gethi(newv));
- stq_be_p(haddr + 1, int128_getlo(newv));
- }
- clear_helper_retaddr();
-#else
int mem_idx = cpu_mmu_index(env, false);
MemOpIdx oi0 = make_memop_idx(MO_BEQ | MO_ALIGN_16, mem_idx);
MemOpIdx oi1 = make_memop_idx(MO_BEQ, mem_idx);
- o1 = helper_be_ldq_mmu(env, addr + 0, oi0, ra);
- o0 = helper_be_ldq_mmu(env, addr + 8, oi1, ra);
+ o1 = cpu_ldq_be_mmu(env, addr + 0, oi0, ra);
+ o0 = cpu_ldq_be_mmu(env, addr + 8, oi1, ra);
oldv = int128_make128(o0, o1);
success = int128_eq(oldv, cmpv);
if (success) {
- helper_be_stq_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
- helper_be_stq_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
+ cpu_stq_be_mmu(env, addr + 0, int128_gethi(newv), oi1, ra);
+ cpu_stq_be_mmu(env, addr + 8, int128_getlo(newv), oi1, ra);
}
-#endif
return !success;
}
diff --git a/target/arm/m_helper.c b/target/arm/m_helper.c
index 62aa12c9d8..2c9922dc29 100644
--- a/target/arm/m_helper.c
+++ b/target/arm/m_helper.c
@@ -1947,9 +1947,9 @@ static bool do_v7m_function_return(ARMCPU *cpu)
* do them as secure, so work out what MMU index that is.
*/
mmu_idx = arm_v7m_mmu_idx_for_secstate(env, true);
- oi = make_memop_idx(MO_LE, arm_to_core_mmu_idx(mmu_idx));
- newpc = helper_le_ldul_mmu(env, frameptr, oi, 0);
- newpsr = helper_le_ldul_mmu(env, frameptr + 4, oi, 0);
+ oi = make_memop_idx(MO_LEUL, arm_to_core_mmu_idx(mmu_idx));
+ newpc = cpu_ldl_le_mmu(env, frameptr, oi, 0);
+ newpsr = cpu_ldl_le_mmu(env, frameptr + 4, oi, 0);
/* Consistency checks on new IPSR */
newpsr_exc = newpsr & XPSR_EXCP;
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 14/15] tcg: Move helper_*_mmu decls to tcg/tcg-ldst.h
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (12 preceding siblings ...)
2021-10-13 18:22 ` [PULL 13/15] target/arm: " Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 18:22 ` [PULL 15/15] tcg: Canonicalize alignment flags in MemOp Richard Henderson
2021-10-13 19:55 ` [PULL 00/15] tcg patch queue Richard Henderson
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé
These functions have been replaced by cpu_*_mmu as the
most proper interface to use from target code.
Hide these declarations from code that should not use them.
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
include/tcg/tcg-ldst.h | 74 ++++++++++++++++++++++++++++++++++++++++++
include/tcg/tcg.h | 71 ----------------------------------------
accel/tcg/cputlb.c | 1 +
tcg/tcg.c | 1 +
tcg/tci.c | 1 +
5 files changed, 77 insertions(+), 71 deletions(-)
create mode 100644 include/tcg/tcg-ldst.h
diff --git a/include/tcg/tcg-ldst.h b/include/tcg/tcg-ldst.h
new file mode 100644
index 0000000000..8c86365611
--- /dev/null
+++ b/include/tcg/tcg-ldst.h
@@ -0,0 +1,74 @@
+/*
+ * Memory helpers that will be used by TCG generated code.
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef TCG_LDST_H
+#define TCG_LDST_H 1
+
+#ifdef CONFIG_SOFTMMU
+
+/* Value zero-extended to tcg register size. */
+tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+
+/* Value sign-extended to tcg register size. */
+tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
+ MemOpIdx oi, uintptr_t retaddr);
+
+void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
+ MemOpIdx oi, uintptr_t retaddr);
+
+#endif /* CONFIG_SOFTMMU */
+#endif /* TCG_LDST_H */
diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h
index 83e38487cf..7069a401f1 100644
--- a/include/tcg/tcg.h
+++ b/include/tcg/tcg.h
@@ -1240,77 +1240,6 @@ uint64_t dup_const(unsigned vece, uint64_t c);
: (target_long)dup_const(VECE, C))
#endif
-/*
- * Memory helpers that will be used by TCG generated code.
- */
-#ifdef CONFIG_SOFTMMU
-/* Value zero-extended to tcg register size. */
-tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-
-/* Value sign-extended to tcg register size. */
-tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, uintptr_t retaddr);
-
-void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
- MemOpIdx oi, uintptr_t retaddr);
-void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- MemOpIdx oi, uintptr_t retaddr);
-void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
- MemOpIdx oi, uintptr_t retaddr);
-void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- MemOpIdx oi, uintptr_t retaddr);
-void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- MemOpIdx oi, uintptr_t retaddr);
-void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
- MemOpIdx oi, uintptr_t retaddr);
-void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- MemOpIdx oi, uintptr_t retaddr);
-
-/* Temporary aliases until backends are converted. */
-#ifdef TARGET_WORDS_BIGENDIAN
-# define helper_ret_ldsw_mmu helper_be_ldsw_mmu
-# define helper_ret_lduw_mmu helper_be_lduw_mmu
-# define helper_ret_ldsl_mmu helper_be_ldsl_mmu
-# define helper_ret_ldul_mmu helper_be_ldul_mmu
-# define helper_ret_ldl_mmu helper_be_ldul_mmu
-# define helper_ret_ldq_mmu helper_be_ldq_mmu
-# define helper_ret_stw_mmu helper_be_stw_mmu
-# define helper_ret_stl_mmu helper_be_stl_mmu
-# define helper_ret_stq_mmu helper_be_stq_mmu
-#else
-# define helper_ret_ldsw_mmu helper_le_ldsw_mmu
-# define helper_ret_lduw_mmu helper_le_lduw_mmu
-# define helper_ret_ldsl_mmu helper_le_ldsl_mmu
-# define helper_ret_ldul_mmu helper_le_ldul_mmu
-# define helper_ret_ldl_mmu helper_le_ldul_mmu
-# define helper_ret_ldq_mmu helper_le_ldq_mmu
-# define helper_ret_stw_mmu helper_le_stw_mmu
-# define helper_ret_stl_mmu helper_le_stl_mmu
-# define helper_ret_stq_mmu helper_le_stq_mmu
-#endif
-#endif /* CONFIG_SOFTMMU */
-
#ifdef CONFIG_DEBUG_TCG
void tcg_assert_listed_vecop(TCGOpcode);
#else
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index b350cafa3d..b69a953447 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -39,6 +39,7 @@
#ifdef CONFIG_PLUGIN
#include "qemu/plugin-memory.h"
#endif
+#include "tcg/tcg-ldst.h"
/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
/* #define DEBUG_TLB */
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 658be0c6b6..024a22cf39 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -58,6 +58,7 @@
#include "elf.h"
#include "exec/log.h"
+#include "tcg/tcg-ldst.h"
#include "tcg-internal.h"
#ifdef CONFIG_TCG_INTERPRETER
diff --git a/tcg/tci.c b/tcg/tci.c
index 5c08dc0a9a..e76087ccac 100644
--- a/tcg/tci.c
+++ b/tcg/tci.c
@@ -22,6 +22,7 @@
#include "tcg/tcg.h" /* MAX_OPC_PARAM_IARGS */
#include "exec/cpu_ldst.h"
#include "tcg/tcg-op.h"
+#include "tcg/tcg-ldst.h"
#include "qemu/compiler.h"
#include <ffi.h>
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* [PULL 15/15] tcg: Canonicalize alignment flags in MemOp
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (13 preceding siblings ...)
2021-10-13 18:22 ` [PULL 14/15] tcg: Move helper_*_mmu decls to tcg/tcg-ldst.h Richard Henderson
@ 2021-10-13 18:22 ` Richard Henderson
2021-10-13 19:55 ` [PULL 00/15] tcg patch queue Richard Henderson
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 18:22 UTC (permalink / raw)
To: qemu-devel; +Cc: Philippe Mathieu-Daudé
Having observed e.g. al8+leq in dumps, canonicalize to al+leq.
Reviewed-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
---
tcg/tcg-op.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index b1cfd36f29..61b492d89f 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -2765,7 +2765,12 @@ void tcg_gen_lookup_and_goto_ptr(void)
static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
{
/* Trigger the asserts within as early as possible. */
- (void)get_alignment_bits(op);
+ unsigned a_bits = get_alignment_bits(op);
+
+ /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
+ if (a_bits == (op & MO_SIZE)) {
+ op = (op & ~MO_AMASK) | MO_ALIGN;
+ }
switch (op & MO_SIZE) {
case MO_8:
--
2.25.1
^ permalink raw reply related [flat|nested] 17+ messages in thread* Re: [PULL 00/15] tcg patch queue
2021-10-13 18:22 [PULL 00/15] tcg patch queue Richard Henderson
` (14 preceding siblings ...)
2021-10-13 18:22 ` [PULL 15/15] tcg: Canonicalize alignment flags in MemOp Richard Henderson
@ 2021-10-13 19:55 ` Richard Henderson
15 siblings, 0 replies; 17+ messages in thread
From: Richard Henderson @ 2021-10-13 19:55 UTC (permalink / raw)
To: qemu-devel
On 10/13/21 11:22 AM, Richard Henderson wrote:
> The following changes since commit ee26ce674a93c824713542cec3b6a9ca85459165:
>
> Merge remote-tracking branch 'remotes/jsnow/tags/python-pull-request' into staging (2021-10-12 16:08:33 -0700)
>
> are available in the Git repository at:
>
> https://gitlab.com/rth7680/qemu.git tags/pull-tcg-20211013
>
> for you to fetch changes up to 76e366e728549b3324cc2dee6745d6a4f1af18e6:
>
> tcg: Canonicalize alignment flags in MemOp (2021-10-13 09:14:35 -0700)
>
> ----------------------------------------------------------------
> Use MO_128 for 16-byte atomic memory operations.
> Add cpu_ld/st_mmu memory primitives.
> Move helper_ld/st memory helpers out of tcg.h.
> Canonicalize alignment flags in MemOp.
>
> ----------------------------------------------------------------
> BALATON Zoltan (1):
> memory: Log access direction for invalid accesses
>
> Richard Henderson (14):
> target/arm: Use MO_128 for 16 byte atomics
> target/i386: Use MO_128 for 16 byte atomics
> target/ppc: Use MO_128 for 16 byte atomics
> target/s390x: Use MO_128 for 16 byte atomics
> target/hexagon: Implement cpu_mmu_index
> accel/tcg: Add cpu_{ld,st}*_mmu interfaces
> accel/tcg: Move cpu_atomic decls to exec/cpu_ldst.h
> target/mips: Use cpu_*_data_ra for msa load/store
> target/mips: Use 8-byte memory ops for msa load/store
> target/s390x: Use cpu_*_mmu instead of helper_*_mmu
> target/sparc: Use cpu_*_mmu instead of helper_*_mmu
> target/arm: Use cpu_*_mmu instead of helper_*_mmu
> tcg: Move helper_*_mmu decls to tcg/tcg-ldst.h
> tcg: Canonicalize alignment flags in MemOp
>
> docs/devel/loads-stores.rst | 52 +++++-
> include/exec/cpu_ldst.h | 332 ++++++++++++++++++-----------------
> include/tcg/tcg-ldst.h | 74 ++++++++
> include/tcg/tcg.h | 158 -----------------
> target/hexagon/cpu.h | 9 +
> accel/tcg/cputlb.c | 393 ++++++++++++++----------------------------
> accel/tcg/user-exec.c | 385 +++++++++++++++++------------------------
> softmmu/memory.c | 20 +--
> target/arm/helper-a64.c | 61 ++-----
> target/arm/m_helper.c | 6 +-
> target/i386/tcg/mem_helper.c | 2 +-
> target/m68k/op_helper.c | 1 -
> target/mips/tcg/msa_helper.c | 389 ++++++++++-------------------------------
> target/ppc/mem_helper.c | 1 -
> target/ppc/translate.c | 12 +-
> target/s390x/tcg/mem_helper.c | 13 +-
> target/sparc/ldst_helper.c | 14 +-
> tcg/tcg-op.c | 7 +-
> tcg/tcg.c | 1 +
> tcg/tci.c | 1 +
> accel/tcg/ldst_common.c.inc | 307 +++++++++++++++++++++++++++++++++
> 21 files changed, 1032 insertions(+), 1206 deletions(-)
> create mode 100644 include/tcg/tcg-ldst.h
> create mode 100644 accel/tcg/ldst_common.c.inc
Applied, thanks.
r~
^ permalink raw reply [flat|nested] 17+ messages in thread