Provide a helper that calls ldst_tlb for all NF vectors.
Signed-off-by: Richard Henderson <[email protected]>
---
target/riscv/tcg/vector_helper.c | 77 +++++++++++++++-----------------
1 file changed, 35 insertions(+), 42 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 8f74162d50..9c0cbcac16 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -329,6 +329,16 @@ static void vext_set_nf_elems_1s(void *vd, uint32_t i,
uint32_t nf,
}
}
+static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
+ uint32_t i, uint32_t nf,
+ uint32_t esz, uint32_t max_elems,
+ vext_ldst_elem_fn_tlb *ldst_tlb, uintptr_t ra)
+{
+ for (uint32_t k = 0; k < nf; k++, addr += esz) {
+ ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ }
+}
+
/*
* stride: access vector element from strided memory
*/
@@ -346,16 +356,11 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base,
target_ulong stride,
VSTART_CHECK_EARLY_EXIT(env, env->vl);
for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
- if (!vm && !vext_elem_mask(v0, i)) {
- if (vma) {
- vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
- }
- continue;
- }
-
- for (uint32_t k = 0; k < nf; ++k) {
- target_ulong addr = base + stride * i + (k << log2_esz);
- ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ if (vm || vext_elem_mask(v0, i)) {
+ vext_ldst_nf_tlb(env, vd, base + stride * i, i, nf,
+ esz, max_elems, ldst_elem, ra);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
}
@@ -411,6 +416,7 @@ vext_page_ldst_us(CPURISCVState *env, void *vd,
target_ulong addr,
int i, k, flags;
uint32_t esz = 1 << log2_esz;
uint32_t size = (elems * nf) << log2_esz;
+ uint32_t msize = nf * esz;
uint32_t evl = env->vstart + elems;
MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
@@ -453,15 +459,10 @@ vext_page_ldst_us(CPURISCVState *env, void *vd,
target_ulong addr,
vext_continuous_ldst_tlb(env, ldst_tlb, vd, evl, addr, env->vstart,
ra, esz, is_load);
} else {
- /* load bytes from guest memory */
for (i = env->vstart; i < evl; env->vstart = ++i) {
- k = 0;
- while (k < nf) {
- ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
- vd, ra);
- addr += esz;
- k++;
- }
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, ra);
+ addr += msize;
}
}
}
@@ -473,7 +474,6 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState
*env, uint32_t desc,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
uint32_t evl, uintptr_t ra, bool is_load)
{
- uint32_t k;
target_ulong page_split, elems, addr;
uint32_t nf = vext_nf(desc);
uint32_t vma = vext_vma(desc);
@@ -519,17 +519,16 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState
*env, uint32_t desc,
/* Load/store elements in the second page */
if (unlikely(env->vstart < evl)) {
+ addr = base + env->vstart * msize;
+
/* Cross page element */
if (unlikely(page_split % msize)) {
- for (k = 0; k < nf; k++) {
- addr = base + ((env->vstart * nf + k) << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr),
- env->vstart + k * max_elems, vd, ra);
- }
+ vext_ldst_nf_tlb(env, vd, addr, env->vstart,
+ nf, esz, max_elems, ldst_tlb, ra);
env->vstart++;
+ addr += msize;
}
- addr = base + ((env->vstart * nf) << log2_esz);
/* Get number of elements of second page */
elems = evl - env->vstart;
@@ -724,7 +723,7 @@ vext_ldff(void *vd, void *v0, target_ulong base,
CPURISCVState *env,
uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra)
{
- uint32_t i, k, vl = 0;
+ uint32_t i, vl = 0;
uint32_t nf = vext_nf(desc);
uint32_t vm = vext_vm(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
@@ -806,17 +805,16 @@ ProbeSuccess:
/* Load/store elements in the second page */
if (unlikely(env->vstart < env->vl)) {
+ addr = base + env->vstart * msize;
+
/* Cross page element */
if (unlikely(page_split % msize)) {
- for (k = 0; k < nf; k++) {
- addr = base + ((env->vstart * nf + k) << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr),
- env->vstart + k * max_elems, vd, ra);
- }
+ vext_ldst_nf_tlb(env, vd, addr, env->vstart, nf,
+ esz, max_elems, ldst_tlb, ra);
env->vstart++;
+ addr += msize;
}
- addr = base + ((env->vstart * nf) << log2_esz);
/* Get number of elements of second page */
elems = env->vl - env->vstart;
@@ -827,16 +825,11 @@ ProbeSuccess:
}
} else {
for (i = env->vstart; i < env->vl; i++) {
- if (!vext_elem_mask(v0, i)) {
- if (vma) {
- vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
- }
- continue;
- }
- for (k = 0; k < nf; ++k) {
- addr = base + ((i * nf + k) << log2_esz);
- ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems,
- vd, ra);
+ if (vext_elem_mask(v0, i)) {
+ vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
+ esz, max_elems, ldst_tlb, ra);
+ } else if (vma) {
+ vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
}
}
--
2.43.0
