All of lore.kernel.org
 help / color / mirror / Atom feed
From: Peter Maydell <peter.maydell@linaro.org>
To: qemu-devel@nongnu.org
Subject: [PULL 087/114] target/arm: Macroize helper_gvec_{s,u}dot_idx_{b,h}
Date: Tue, 25 May 2021 16:02:57 +0100	[thread overview]
Message-ID: <20210525150324.32370-88-peter.maydell@linaro.org> (raw)
In-Reply-To: <20210525150324.32370-1-peter.maydell@linaro.org>

From: Richard Henderson <richard.henderson@linaro.org>

We're about to add more variations on this theme.
Accept the inner loop for the _h variants, rather
than keep it unrolled.

Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20210525010358.152808-66-richard.henderson@linaro.org
Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
---
 target/arm/vec_helper.c | 160 ++++++++--------------------------------
 1 file changed, 29 insertions(+), 131 deletions(-)

diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c
index cddf095c74a..1c4266a9c09 100644
--- a/target/arm/vec_helper.c
+++ b/target/arm/vec_helper.c
@@ -569,139 +569,37 @@ DO_DOT(gvec_udot_b, uint32_t, uint8_t, uint8_t)
 DO_DOT(gvec_sdot_h, int64_t, int16_t, int16_t)
 DO_DOT(gvec_udot_h, uint64_t, uint16_t, uint16_t)
 
-void HELPER(gvec_sdot_idx_b)(void *vd, void *vn, void *vm,
-                             void *va, uint32_t desc)
-{
-    intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
-    intptr_t index = simd_data(desc);
-    int32_t *d = vd, *a = va;
-    int8_t *n = vn;
-    int8_t *m_indexed = (int8_t *)vm + H4(index) * 4;
-
-    /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
-     * Otherwise opr_sz is a multiple of 16.
-     */
-    segend = MIN(4, opr_sz_4);
-    i = 0;
-    do {
-        int8_t m0 = m_indexed[i * 4 + 0];
-        int8_t m1 = m_indexed[i * 4 + 1];
-        int8_t m2 = m_indexed[i * 4 + 2];
-        int8_t m3 = m_indexed[i * 4 + 3];
-
-        do {
-            d[i] = (a[i] +
-                    n[i * 4 + 0] * m0 +
-                    n[i * 4 + 1] * m1 +
-                    n[i * 4 + 2] * m2 +
-                    n[i * 4 + 3] * m3);
-        } while (++i < segend);
-        segend = i + 4;
-    } while (i < opr_sz_4);
-
-    clear_tail(d, opr_sz, simd_maxsz(desc));
+#define DO_DOT_IDX(NAME, TYPED, TYPEN, TYPEM, HD) \
+void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc)  \
+{                                                                         \
+    intptr_t i = 0, opr_sz = simd_oprsz(desc);                            \
+    intptr_t opr_sz_n = opr_sz / sizeof(TYPED);                           \
+    intptr_t segend = MIN(16 / sizeof(TYPED), opr_sz_n);                  \
+    intptr_t index = simd_data(desc);                                     \
+    TYPED *d = vd, *a = va;                                               \
+    TYPEN *n = vn;                                                        \
+    TYPEM *m_indexed = (TYPEM *)vm + HD(index) * 4;                       \
+    do {                                                                  \
+        TYPED m0 = m_indexed[i * 4 + 0];                                  \
+        TYPED m1 = m_indexed[i * 4 + 1];                                  \
+        TYPED m2 = m_indexed[i * 4 + 2];                                  \
+        TYPED m3 = m_indexed[i * 4 + 3];                                  \
+        do {                                                              \
+            d[i] = (a[i] +                                                \
+                    n[i * 4 + 0] * m0 +                                   \
+                    n[i * 4 + 1] * m1 +                                   \
+                    n[i * 4 + 2] * m2 +                                   \
+                    n[i * 4 + 3] * m3);                                   \
+        } while (++i < segend);                                           \
+        segend = i + 4;                                                   \
+    } while (i < opr_sz_n);                                               \
+    clear_tail(d, opr_sz, simd_maxsz(desc));                              \
 }
 
-void HELPER(gvec_udot_idx_b)(void *vd, void *vn, void *vm,
-                             void *va, uint32_t desc)
-{
-    intptr_t i, segend, opr_sz = simd_oprsz(desc), opr_sz_4 = opr_sz / 4;
-    intptr_t index = simd_data(desc);
-    uint32_t *d = vd, *a = va;
-    uint8_t *n = vn;
-    uint8_t *m_indexed = (uint8_t *)vm + H4(index) * 4;
-
-    /* Notice the special case of opr_sz == 8, from aa64/aa32 advsimd.
-     * Otherwise opr_sz is a multiple of 16.
-     */
-    segend = MIN(4, opr_sz_4);
-    i = 0;
-    do {
-        uint8_t m0 = m_indexed[i * 4 + 0];
-        uint8_t m1 = m_indexed[i * 4 + 1];
-        uint8_t m2 = m_indexed[i * 4 + 2];
-        uint8_t m3 = m_indexed[i * 4 + 3];
-
-        do {
-            d[i] = (a[i] +
-                    n[i * 4 + 0] * m0 +
-                    n[i * 4 + 1] * m1 +
-                    n[i * 4 + 2] * m2 +
-                    n[i * 4 + 3] * m3);
-        } while (++i < segend);
-        segend = i + 4;
-    } while (i < opr_sz_4);
-
-    clear_tail(d, opr_sz, simd_maxsz(desc));
-}
-
-void HELPER(gvec_sdot_idx_h)(void *vd, void *vn, void *vm,
-                             void *va, uint32_t desc)
-{
-    intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
-    intptr_t index = simd_data(desc);
-    int64_t *d = vd, *a = va;
-    int16_t *n = vn;
-    int16_t *m_indexed = (int16_t *)vm + index * 4;
-
-    /* This is supported by SVE only, so opr_sz is always a multiple of 16.
-     * Process the entire segment all at once, writing back the results
-     * only after we've consumed all of the inputs.
-     */
-    for (i = 0; i < opr_sz_8; i += 2) {
-        int64_t d0, d1;
-
-        d0  = a[i + 0];
-        d0 += n[i * 4 + 0] * (int64_t)m_indexed[i * 4 + 0];
-        d0 += n[i * 4 + 1] * (int64_t)m_indexed[i * 4 + 1];
-        d0 += n[i * 4 + 2] * (int64_t)m_indexed[i * 4 + 2];
-        d0 += n[i * 4 + 3] * (int64_t)m_indexed[i * 4 + 3];
-
-        d1  = a[i + 1];
-        d1 += n[i * 4 + 4] * (int64_t)m_indexed[i * 4 + 0];
-        d1 += n[i * 4 + 5] * (int64_t)m_indexed[i * 4 + 1];
-        d1 += n[i * 4 + 6] * (int64_t)m_indexed[i * 4 + 2];
-        d1 += n[i * 4 + 7] * (int64_t)m_indexed[i * 4 + 3];
-
-        d[i + 0] = d0;
-        d[i + 1] = d1;
-    }
-    clear_tail(d, opr_sz, simd_maxsz(desc));
-}
-
-void HELPER(gvec_udot_idx_h)(void *vd, void *vn, void *vm,
-                             void *va, uint32_t desc)
-{
-    intptr_t i, opr_sz = simd_oprsz(desc), opr_sz_8 = opr_sz / 8;
-    intptr_t index = simd_data(desc);
-    uint64_t *d = vd, *a = va;
-    uint16_t *n = vn;
-    uint16_t *m_indexed = (uint16_t *)vm + index * 4;
-
-    /* This is supported by SVE only, so opr_sz is always a multiple of 16.
-     * Process the entire segment all at once, writing back the results
-     * only after we've consumed all of the inputs.
-     */
-    for (i = 0; i < opr_sz_8; i += 2) {
-        uint64_t d0, d1;
-
-        d0  = a[i + 0];
-        d0 += n[i * 4 + 0] * (uint64_t)m_indexed[i * 4 + 0];
-        d0 += n[i * 4 + 1] * (uint64_t)m_indexed[i * 4 + 1];
-        d0 += n[i * 4 + 2] * (uint64_t)m_indexed[i * 4 + 2];
-        d0 += n[i * 4 + 3] * (uint64_t)m_indexed[i * 4 + 3];
-
-        d1  = a[i + 1];
-        d1 += n[i * 4 + 4] * (uint64_t)m_indexed[i * 4 + 0];
-        d1 += n[i * 4 + 5] * (uint64_t)m_indexed[i * 4 + 1];
-        d1 += n[i * 4 + 6] * (uint64_t)m_indexed[i * 4 + 2];
-        d1 += n[i * 4 + 7] * (uint64_t)m_indexed[i * 4 + 3];
-
-        d[i + 0] = d0;
-        d[i + 1] = d1;
-    }
-    clear_tail(d, opr_sz, simd_maxsz(desc));
-}
+DO_DOT_IDX(gvec_sdot_idx_b, int32_t, int8_t, int8_t, H4)
+DO_DOT_IDX(gvec_udot_idx_b, uint32_t, uint8_t, uint8_t, H4)
+DO_DOT_IDX(gvec_sdot_idx_h, int64_t, int16_t, int16_t, )
+DO_DOT_IDX(gvec_udot_idx_h, uint64_t, uint16_t, uint16_t, )
 
 void HELPER(gvec_fcaddh)(void *vd, void *vn, void *vm,
                          void *vfpst, uint32_t desc)
-- 
2.20.1



  parent reply	other threads:[~2021-05-25 16:12 UTC|newest]

Thread overview: 96+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-25 15:01 [PULL 000/114] target-arm queue Peter Maydell
2021-05-25 15:01 ` [PULL 001/114] hw/arm/smmuv3: Another range invalidation fix Peter Maydell
2021-05-25 15:01 ` [PULL 002/114] hw/intc/arm_gicv3_cpuif: Fix EOIR write access check logic Peter Maydell
2021-05-25 15:01 ` [PULL 003/114] hw/arm/mps2-tz: Don't duplicate modelling of SRAM in AN524 Peter Maydell
2021-05-25 15:01 ` [PULL 004/114] hw/arm/mps2-tz: Make SRAM_ADDR_WIDTH board-specific Peter Maydell
2021-05-25 15:01 ` [PULL 005/114] hw/arm/armsse.c: Correct modelling of SSE-300 internal SRAMs Peter Maydell
2021-05-25 15:01 ` [PULL 006/114] hw/arm/armsse: Convert armsse_realize() to use ERRP_GUARD Peter Maydell
2021-05-25 15:01 ` [PULL 007/114] hw/arm/mps2-tz: Allow board to specify a boot RAM size Peter Maydell
2021-05-25 15:01 ` [PULL 008/114] hw/arm: Model TCMs in the SSE-300, not the AN547 Peter Maydell
2021-05-25 15:01 ` [PULL 009/114] target/arm: Use correct SP in M-profile exception return Peter Maydell
2021-05-25 15:01 ` [PULL 010/114] accel/tcg: Replace g_new() + memcpy() by g_memdup() Peter Maydell
2021-05-25 15:01 ` [PULL 011/114] accel/tcg: Pass length argument to tlb_flush_range_locked() Peter Maydell
2021-05-25 15:01 ` [PULL 012/114] accel/tlb: Rename TLBFlushPageBitsByMMUIdxData -> TLBFlushRangeData Peter Maydell
2021-05-25 15:01 ` [PULL 013/114] accel/tcg: Remove {encode,decode}_pbm_to_runon Peter Maydell
2021-05-25 15:01 ` [PULL 014/114] accel/tcg: Add tlb_flush_range_by_mmuidx() Peter Maydell
2021-05-25 15:01 ` [PULL 015/114] accel/tcg: Add tlb_flush_range_by_mmuidx_all_cpus() Peter Maydell
2021-05-25 15:01 ` [PULL 016/114] accel/tlb: Add tlb_flush_range_by_mmuidx_all_cpus_synced() Peter Maydell
2021-05-25 15:01 ` [PULL 017/114] accel/tcg: Rename tlb_flush_page_bits -> range]_by_mmuidx_async_0 Peter Maydell
2021-05-25 15:01 ` [PULL 018/114] accel/tlb: Rename tlb_flush_[page_bits > range]_by_mmuidx_async_[2 > 1] Peter Maydell
2021-05-25 15:01 ` [PULL 019/114] target/arm: Add support for FEAT_TLBIRANGE Peter Maydell
2021-05-25 15:01 ` [PULL 020/114] target/arm: Add support for FEAT_TLBIOS Peter Maydell
2021-05-25 15:01 ` [PULL 021/114] target/arm: set ID_AA64ISAR0.TLB to 2 for max AARCH64 CPU type Peter Maydell
2021-05-25 15:01 ` [PULL 022/114] disas/libvixl: Protect C system header for C++ compiler Peter Maydell
2021-05-25 15:01 ` [PULL 023/114] target/arm: Add ID_AA64ZFR0 fields and isar_feature_aa64_sve2 Peter Maydell
2021-05-25 15:01 ` [PULL 024/114] target/arm: Implement SVE2 Integer Multiply - Unpredicated Peter Maydell
2021-05-25 15:01 ` [PULL 025/114] target/arm: Implement SVE2 integer pairwise add and accumulate long Peter Maydell
2021-05-25 15:01 ` [PULL 026/114] target/arm: Implement SVE2 integer unary operations (predicated) Peter Maydell
2021-05-25 15:01 ` [PULL 027/114] target/arm: Split out saturating/rounding shifts from neon Peter Maydell
2021-05-25 15:01 ` [PULL 028/114] target/arm: Implement SVE2 saturating/rounding bitwise shift left (predicated) Peter Maydell
2021-05-25 15:01 ` [PULL 029/114] target/arm: Implement SVE2 integer halving add/subtract (predicated) Peter Maydell
2021-05-25 15:02 ` [PULL 030/114] target/arm: Implement SVE2 integer pairwise arithmetic Peter Maydell
2021-05-25 15:02 ` [PULL 031/114] target/arm: Implement SVE2 saturating add/subtract (predicated) Peter Maydell
2021-05-25 15:02 ` [PULL 032/114] target/arm: Implement SVE2 integer add/subtract long Peter Maydell
2021-05-25 15:02 ` [PULL 033/114] target/arm: Implement SVE2 integer add/subtract interleaved long Peter Maydell
2021-05-25 15:02 ` [PULL 034/114] target/arm: Implement SVE2 integer add/subtract wide Peter Maydell
2021-05-25 15:02 ` [PULL 035/114] target/arm: Implement SVE2 integer multiply long Peter Maydell
2021-05-25 15:02 ` [PULL 036/114] target/arm: Implement SVE2 PMULLB, PMULLT Peter Maydell
2021-05-25 15:02 ` [PULL 037/114] target/arm: Implement SVE2 bitwise shift left long Peter Maydell
2021-05-25 15:02 ` [PULL 038/114] target/arm: Implement SVE2 bitwise exclusive-or interleaved Peter Maydell
2021-05-25 15:02 ` [PULL 039/114] target/arm: Implement SVE2 bitwise permute Peter Maydell
2021-05-25 15:02 ` [PULL 040/114] target/arm: Implement SVE2 complex integer add Peter Maydell
2021-05-25 15:02 ` [PULL 041/114] target/arm: Implement SVE2 integer absolute difference and accumulate long Peter Maydell
2021-05-25 15:02 ` [PULL 042/114] target/arm: Implement SVE2 integer add/subtract long with carry Peter Maydell
2021-05-25 15:02 ` [PULL 043/114] target/arm: Implement SVE2 bitwise shift right and accumulate Peter Maydell
2021-05-25 15:02 ` [PULL 044/114] target/arm: Implement SVE2 bitwise shift and insert Peter Maydell
2021-05-25 15:02 ` [PULL 045/114] target/arm: Implement SVE2 integer absolute difference and accumulate Peter Maydell
2021-05-25 15:02 ` [PULL 046/114] target/arm: Implement SVE2 saturating extract narrow Peter Maydell
2021-05-25 15:02 ` [PULL 047/114] target/arm: Implement SVE2 floating-point pairwise Peter Maydell
2021-05-25 15:02 ` [PULL 048/114] target/arm: Implement SVE2 SHRN, RSHRN Peter Maydell
2021-05-25 15:02 ` [PULL 049/114] target/arm: Implement SVE2 SQSHRUN, SQRSHRUN Peter Maydell
2021-05-25 15:02 ` [PULL 050/114] target/arm: Implement SVE2 UQSHRN, UQRSHRN Peter Maydell
2021-05-25 15:02 ` [PULL 051/114] target/arm: Implement SVE2 SQSHRN, SQRSHRN Peter Maydell
2021-05-25 15:02 ` [PULL 052/114] target/arm: Implement SVE2 WHILEGT, WHILEGE, WHILEHI, WHILEHS Peter Maydell
2021-05-25 15:02 ` [PULL 053/114] target/arm: Implement SVE2 WHILERW, WHILEWR Peter Maydell
2021-05-25 15:02 ` [PULL 054/114] target/arm: Implement SVE2 bitwise ternary operations Peter Maydell
2021-05-25 15:02 ` [PULL 055/114] target/arm: Implement SVE2 MATCH, NMATCH Peter Maydell
2021-05-25 15:02 ` [PULL 056/114] target/arm: Implement SVE2 saturating multiply-add long Peter Maydell
2021-05-25 15:02 ` [PULL 057/114] target/arm: Implement SVE2 saturating multiply-add high Peter Maydell
2021-05-25 15:02 ` [PULL 058/114] target/arm: Implement SVE2 integer multiply-add long Peter Maydell
2021-05-25 15:02 ` [PULL 059/114] target/arm: Implement SVE2 complex integer multiply-add Peter Maydell
2021-05-25 15:02 ` [PULL 060/114] target/arm: Implement SVE2 ADDHNB, ADDHNT Peter Maydell
2021-05-25 15:02 ` [PULL 061/114] target/arm: Implement SVE2 RADDHNB, RADDHNT Peter Maydell
2021-05-25 15:02 ` [PULL 062/114] target/arm: Implement SVE2 SUBHNB, SUBHNT Peter Maydell
2021-05-25 15:02 ` [PULL 063/114] target/arm: Implement SVE2 RSUBHNB, RSUBHNT Peter Maydell
2021-05-25 15:02 ` [PULL 064/114] target/arm: Implement SVE2 HISTCNT, HISTSEG Peter Maydell
2021-05-25 15:02 ` [PULL 065/114] target/arm: Implement SVE2 XAR Peter Maydell
2021-05-25 15:02 ` [PULL 066/114] target/arm: Implement SVE2 scatter store insns Peter Maydell
2021-05-25 15:02 ` [PULL 067/114] target/arm: Implement SVE2 gather load insns Peter Maydell
2021-05-25 15:02 ` [PULL 068/114] target/arm: Implement SVE2 FMMLA Peter Maydell
2021-05-25 15:02 ` [PULL 069/114] target/arm: Implement SVE2 SPLICE, EXT Peter Maydell
2021-05-25 15:02 ` [PULL 070/114] target/arm: Use correct output type for gvec_sdot_*_b Peter Maydell
2021-05-25 15:02 ` [PULL 071/114] target/arm: Pass separate addend to {U, S}DOT helpers Peter Maydell
2021-05-25 15:02 ` [PULL 072/114] target/arm: Pass separate addend to FCMLA helpers Peter Maydell
2021-05-25 15:02 ` [PULL 073/114] target/arm: Split out formats for 2 vectors + 1 index Peter Maydell
2021-05-25 15:02 ` [PULL 074/114] target/arm: Split out formats for 3 " Peter Maydell
2021-05-25 15:02 ` [PULL 075/114] target/arm: Implement SVE2 integer multiply (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 076/114] target/arm: Implement SVE2 integer multiply-add (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 077/114] target/arm: Implement SVE2 saturating multiply-add high (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 078/114] target/arm: Implement SVE2 saturating multiply-add (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 079/114] target/arm: Implement SVE2 saturating multiply (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 080/114] target/arm: Implement SVE2 signed saturating doubling multiply high Peter Maydell
2021-05-25 15:02 ` [PULL 081/114] target/arm: Implement SVE2 saturating multiply high (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 082/114] target/arm: Implement SVE2 multiply-add long (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 083/114] target/arm: Implement SVE2 integer multiply " Peter Maydell
2021-05-25 15:02 ` [PULL 084/114] target/arm: Implement SVE2 complex integer multiply-add (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 085/114] target/arm: Implement SVE2 complex integer dot product Peter Maydell
2021-05-25 15:02 ` [PULL 086/114] target/arm: Macroize helper_gvec_{s,u}dot_{b,h} Peter Maydell
2021-05-25 15:02 ` Peter Maydell [this message]
2021-05-25 15:02 ` [PULL 088/114] target/arm: Implement SVE mixed sign dot product (indexed) Peter Maydell
2021-05-25 15:02 ` [PULL 089/114] target/arm: Implement SVE mixed sign dot product Peter Maydell
2021-05-25 15:03 ` [PULL 090/114] target/arm: Implement SVE2 crypto unary operations Peter Maydell
2021-05-25 15:03 ` [PULL 091/114] target/arm: Implement SVE2 crypto destructive binary operations Peter Maydell
2021-05-25 15:03 ` [PULL 092/114] target/arm: Implement SVE2 crypto constructive " Peter Maydell
2021-05-25 15:03 ` [PULL 093/114] target/arm: Implement SVE2 TBL, TBX Peter Maydell
2021-05-25 15:03 ` [PULL 094/114] target/arm: Implement SVE2 FCVTNT Peter Maydell
2021-05-25 16:30 ` [PULL 000/114] target-arm queue Peter Maydell

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210525150324.32370-88-peter.maydell@linaro.org \
    --to=peter.maydell@linaro.org \
    --cc=qemu-devel@nongnu.org \
    --subject='Re: [PULL 087/114] target/arm: Macroize helper_gvec_{s,u}dot_idx_{b,h}' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.