This merges vext_continuous_ldst_host into the ldst_host function. We can then handle the byte little-endian optimization at compile-time.
Signed-off-by: Richard Henderson <[email protected]> --- target/riscv/tcg/vector_helper.c | 83 +++++++++++++++----------------- 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index 661f1ab2f6..3786a17231 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -214,7 +214,8 @@ static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size) /* elements operations for load and store */ typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, void *vd, uintptr_t retaddr); -typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host); +typedef void vext_ldst_elem_fn_host(void *vd, void *host, + uint32_t idx, uint32_t evl); #define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \ static inline QEMU_ALWAYS_INLINE \ @@ -226,12 +227,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \ } \ -#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ -static inline QEMU_ALWAYS_INLINE \ -void NAME##_host(void *vd, uint32_t idx, void *host) \ -{ \ - ETYPE *cur = ((ETYPE *)vd + H(idx)); \ - *cur = (ETYPE)LDSUF##_p(host); \ +#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ +static inline QEMU_ALWAYS_INLINE \ +void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \ +{ \ + do { \ + ETYPE *cur = (ETYPE *)vd + H(idx); \ + *cur = LDSUF##_p(host); \ + host += sizeof(ETYPE); \ + } while (++idx < evl); \ } GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb) @@ -239,7 +243,16 @@ GEN_VEXT_TLB_LD_ELEM(lde_h, uint16_t, H2, ldw) GEN_VEXT_TLB_LD_ELEM(lde_w, uint32_t, H4, ldl) GEN_VEXT_TLB_LD_ELEM(lde_d, uint64_t, H8, ldq) +#if HOST_BIG_ENDIAN GEN_VEXT_HOST_LD_ELEM(lde_b, uint8_t, H1, ldub) +#else +static inline QEMU_ALWAYS_INLINE +void lde_b_host(void *vd, void *host, uint32_t idx, uint32_t evl) +{ + memcpy(vd + idx, host, evl - idx); +} +#endif + GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le) GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le) GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le) @@ -254,12 +267,15 @@ void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \ } \ -#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ -static inline QEMU_ALWAYS_INLINE \ -void NAME##_host(void *vd, uint32_t idx, void *host) \ -{ \ - ETYPE data = *((ETYPE *)vd + H(idx)); \ - STSUF##_p(host, data); \ +#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ +static inline QEMU_ALWAYS_INLINE \ +void NAME##_host(void *vd, void *host, uint32_t idx, uint32_t evl) \ +{ \ + do { \ + ETYPE data = *((ETYPE *)vd + H(idx)); \ + STSUF##_p(host, data); \ + host += sizeof(ETYPE); \ + } while (++idx < evl); \ } GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb) @@ -267,7 +283,16 @@ GEN_VEXT_TLB_ST_ELEM(ste_h, uint16_t, H2, stw) GEN_VEXT_TLB_ST_ELEM(ste_w, uint32_t, H4, stl) GEN_VEXT_TLB_ST_ELEM(ste_d, uint64_t, H8, stq) +#if HOST_BIG_ENDIAN GEN_VEXT_HOST_ST_ELEM(ste_b, uint8_t, H1, stb) +#else +static inline QEMU_ALWAYS_INLINE +void ste_b_host(void *vd, void *host, uint32_t idx, uint32_t evl) +{ + memcpy(host, vd + idx, evl - idx); +} +#endif + GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le) GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le) GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le) @@ -284,33 +309,6 @@ vext_continuous_ldst_tlb(CPURISCVState *env, vext_ldst_elem_fn_tlb *ldst_tlb, } } -static inline QEMU_ALWAYS_INLINE void -vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host, - void *vd, uint32_t evl, uint32_t reg_start, void *host, - uint32_t esz, bool is_load) -{ - if (HOST_BIG_ENDIAN) { - for (; reg_start < evl; reg_start++, host += esz) { - ldst_host(vd, reg_start, host); - } - } else { - if (esz == 1) { - uint32_t byte_offset = reg_start * esz; - uint32_t size = (evl - reg_start) * esz; - - if (is_load) { - memcpy(vd + byte_offset, host, size); - } else { - memcpy(host, vd + byte_offset, size); - } - } else { - for (; reg_start < evl; reg_start++, host += esz) { - ldst_host(vd, reg_start, host); - } - } - } -} - static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t nf, uint32_t esz, uint32_t max_elems) { @@ -334,7 +332,7 @@ static void vext_ldst_nf_host(void *vd, void *host, uint32_t i, uint32_t nf, vext_ldst_elem_fn_host *ldst_host) { for (uint32_t k = 0; k < nf; k++, host += esz) { - ldst_host(vd, i + k * max_elems, host); + ldst_host(vd + k * max_elems, host, i, i + 1); } } @@ -450,8 +448,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr, */ if (flags == 0 && (riscv_cpu_cfg(env)->ext_zicclsm || !misaligned)) { if (nf == 1) { - vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart, - host, esz, is_load); + ldst_host(vd, host, env->vstart, evl); } else { for (uint32_t i = env->vstart; i < evl; ++i) { vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host); -- 2.43.0
