From: Taylor Simpson <tsimpson@quicinc.com>
To: qemu-devel@nongnu.org
Cc: tsimpson@quicinc.com, richard.henderson@linaro.org,
philmd@linaro.org, ale@rev.ng, anjo@rev.ng, bcain@quicinc.com,
quic_mathbern@quicinc.com
Subject: [PATCH v3 8/9] Hexagon (tests/tcg/hexagon) Remove __builtin from scatter_gather
Date: Thu, 5 Jan 2023 14:13:30 -0800 [thread overview]
Message-ID: <20230105221331.12069-9-tsimpson@quicinc.com> (raw)
In-Reply-To: <20230105221331.12069-1-tsimpson@quicinc.com>
Replace __builtin_* with inline assembly
The __builtin's are subject to change with different compiler
releases, so might break
Mark arrays as aligned when accessed as HVX vectors
Clean up comments
Signed-off-by: Taylor Simpson <tsimpson@quicinc.com>
---
tests/tcg/hexagon/scatter_gather.c | 513 +++++++++++++++--------------
1 file changed, 271 insertions(+), 242 deletions(-)
diff --git a/tests/tcg/hexagon/scatter_gather.c b/tests/tcg/hexagon/scatter_gather.c
index b93eb18133..bf8b5e0317 100644
--- a/tests/tcg/hexagon/scatter_gather.c
+++ b/tests/tcg/hexagon/scatter_gather.c
@@ -1,5 +1,5 @@
/*
- * Copyright(c) 2019-2021 Qualcomm Innovation Center, Inc. All Rights Reserved.
+ * Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -40,47 +40,6 @@ typedef long HVX_VectorPair __attribute__((__vector_size__(256)))
typedef long HVX_VectorPred __attribute__((__vector_size__(128)))
__attribute__((aligned(128)));
-#define VSCATTER_16(BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermh_128B((int)BASE, RGN, OFF, VALS)
-#define VSCATTER_16_MASKED(MASK, BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermhq_128B(MASK, (int)BASE, RGN, OFF, VALS)
-#define VSCATTER_32(BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermw_128B((int)BASE, RGN, OFF, VALS)
-#define VSCATTER_32_MASKED(MASK, BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermwq_128B(MASK, (int)BASE, RGN, OFF, VALS)
-#define VSCATTER_16_32(BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermhw_128B((int)BASE, RGN, OFF, VALS)
-#define VSCATTER_16_32_MASKED(MASK, BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermhwq_128B(MASK, (int)BASE, RGN, OFF, VALS)
-#define VSCATTER_16_ACC(BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermh_add_128B((int)BASE, RGN, OFF, VALS)
-#define VSCATTER_32_ACC(BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermw_add_128B((int)BASE, RGN, OFF, VALS)
-#define VSCATTER_16_32_ACC(BASE, RGN, OFF, VALS) \
- __builtin_HEXAGON_V6_vscattermhw_add_128B((int)BASE, RGN, OFF, VALS)
-
-#define VGATHER_16(DSTADDR, BASE, RGN, OFF) \
- __builtin_HEXAGON_V6_vgathermh_128B(DSTADDR, (int)BASE, RGN, OFF)
-#define VGATHER_16_MASKED(DSTADDR, MASK, BASE, RGN, OFF) \
- __builtin_HEXAGON_V6_vgathermhq_128B(DSTADDR, MASK, (int)BASE, RGN, OFF)
-#define VGATHER_32(DSTADDR, BASE, RGN, OFF) \
- __builtin_HEXAGON_V6_vgathermw_128B(DSTADDR, (int)BASE, RGN, OFF)
-#define VGATHER_32_MASKED(DSTADDR, MASK, BASE, RGN, OFF) \
- __builtin_HEXAGON_V6_vgathermwq_128B(DSTADDR, MASK, (int)BASE, RGN, OFF)
-#define VGATHER_16_32(DSTADDR, BASE, RGN, OFF) \
- __builtin_HEXAGON_V6_vgathermhw_128B(DSTADDR, (int)BASE, RGN, OFF)
-#define VGATHER_16_32_MASKED(DSTADDR, MASK, BASE, RGN, OFF) \
- __builtin_HEXAGON_V6_vgathermhwq_128B(DSTADDR, MASK, (int)BASE, RGN, OFF)
-
-#define VSHUFF_H(V) \
- __builtin_HEXAGON_V6_vshuffh_128B(V)
-#define VSPLAT_H(X) \
- __builtin_HEXAGON_V6_lvsplath_128B(X)
-#define VAND_VAL(PRED, VAL) \
- __builtin_HEXAGON_V6_vandvrt_128B(PRED, VAL)
-#define VDEAL_H(V) \
- __builtin_HEXAGON_V6_vdealh_128B(V)
-
int err;
/* define the number of rows/cols in a square matrix */
@@ -108,22 +67,22 @@ unsigned short vscatter16_32_ref[SCATTER_BUFFER_SIZE];
unsigned short vgather16_32_ref[MATRIX_SIZE];
/* declare the arrays of offsets */
-unsigned short half_offsets[MATRIX_SIZE];
-unsigned int word_offsets[MATRIX_SIZE];
+unsigned short half_offsets[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned int word_offsets[MATRIX_SIZE] __attribute__((aligned(128)));
/* declare the arrays of values */
-unsigned short half_values[MATRIX_SIZE];
-unsigned short half_values_acc[MATRIX_SIZE];
-unsigned short half_values_masked[MATRIX_SIZE];
-unsigned int word_values[MATRIX_SIZE];
-unsigned int word_values_acc[MATRIX_SIZE];
-unsigned int word_values_masked[MATRIX_SIZE];
+unsigned short half_values[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned short half_values_acc[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned short half_values_masked[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned int word_values[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned int word_values_acc[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned int word_values_masked[MATRIX_SIZE] __attribute__((aligned(128)));
/* declare the arrays of predicates */
-unsigned short half_predicates[MATRIX_SIZE];
-unsigned int word_predicates[MATRIX_SIZE];
+unsigned short half_predicates[MATRIX_SIZE] __attribute__((aligned(128)));
+unsigned int word_predicates[MATRIX_SIZE] __attribute__((aligned(128)));
-/* make this big enough for all the intrinsics */
+/* make this big enough for all the operations */
const size_t region_len = sizeof(vtcm);
/* optionally add sync instructions */
@@ -261,164 +220,201 @@ void create_offsets_values_preds_16_32(void)
}
}
-/* scatter the 16 bit elements using intrinsics */
+/* scatter the 16 bit elements using HVX */
void vector_scatter_16(void)
{
- /* copy the offsets and values to vectors */
- HVX_Vector offsets = *(HVX_Vector *)half_offsets;
- HVX_Vector values = *(HVX_Vector *)half_values;
-
- VSCATTER_16(&vtcm.vscatter16, region_len, offsets, values);
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%3 + #0)\n\t"
+ "vscatter(%0, m0, v0.h).h = v1\n\t"
+ : : "r"(vtcm.vscatter16), "r"(region_len),
+ "r"(half_offsets), "r"(half_values)
+ : "m0", "v0", "v1", "memory");
sync_scatter(vtcm.vscatter16);
}
-/* scatter-accumulate the 16 bit elements using intrinsics */
+/* scatter-accumulate the 16 bit elements using HVX */
void vector_scatter_16_acc(void)
{
- /* copy the offsets and values to vectors */
- HVX_Vector offsets = *(HVX_Vector *)half_offsets;
- HVX_Vector values = *(HVX_Vector *)half_values_acc;
-
- VSCATTER_16_ACC(&vtcm.vscatter16, region_len, offsets, values);
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%3 + #0)\n\t"
+ "vscatter(%0, m0, v0.h).h += v1\n\t"
+ : : "r"(vtcm.vscatter16), "r"(region_len),
+ "r"(half_offsets), "r"(half_values_acc)
+ : "m0", "v0", "v1", "memory");
sync_scatter(vtcm.vscatter16);
}
-/* scatter the 16 bit elements using intrinsics */
+/* masked scatter the 16 bit elements using HVX */
void vector_scatter_16_masked(void)
{
- /* copy the offsets and values to vectors */
- HVX_Vector offsets = *(HVX_Vector *)half_offsets;
- HVX_Vector values = *(HVX_Vector *)half_values_masked;
- HVX_Vector pred_reg = *(HVX_Vector *)half_predicates;
- HVX_VectorPred preds = VAND_VAL(pred_reg, ~0);
-
- VSCATTER_16_MASKED(preds, &vtcm.vscatter16, region_len, offsets, values);
+ asm ("r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "v1 = vmem(%4 + #0)\n\t"
+ "if (q0) vscatter(%1, m0, v0.h).h = v1\n\t"
+ : : "r"(half_predicates), "r"(vtcm.vscatter16), "r"(region_len),
+ "r"(half_offsets), "r"(half_values_masked)
+ : "r1", "q0", "m0", "q0", "v0", "v1", "memory");
sync_scatter(vtcm.vscatter16);
}
-/* scatter the 32 bit elements using intrinsics */
+/* scatter the 32 bit elements using HVX */
void vector_scatter_32(void)
{
- /* copy the offsets and values to vectors */
- HVX_Vector offsetslo = *(HVX_Vector *)word_offsets;
- HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
- HVX_Vector valueslo = *(HVX_Vector *)word_values;
- HVX_Vector valueshi = *(HVX_Vector *)&word_values[MATRIX_SIZE / 2];
-
- VSCATTER_32(&vtcm.vscatter32, region_len, offsetslo, valueslo);
- VSCATTER_32(&vtcm.vscatter32, region_len, offsetshi, valueshi);
+ HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
+ HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
+ HVX_Vector *valueslo = (HVX_Vector *)word_values;
+ HVX_Vector *valueshi = (HVX_Vector *)&word_values[MATRIX_SIZE / 2];
+
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%3 + #0)\n\t"
+ "vscatter(%0, m0, v0.w).w = v1\n\t"
+ : : "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetslo), "r"(valueslo)
+ : "m0", "v0", "v1", "memory");
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%3 + #0)\n\t"
+ "vscatter(%0, m0, v0.w).w = v1\n\t"
+ : : "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetshi), "r"(valueshi)
+ : "m0", "v0", "v1", "memory");
sync_scatter(vtcm.vscatter32);
}
-/* scatter-acc the 32 bit elements using intrinsics */
+/* scatter-accumulate the 32 bit elements using HVX */
void vector_scatter_32_acc(void)
{
- /* copy the offsets and values to vectors */
- HVX_Vector offsetslo = *(HVX_Vector *)word_offsets;
- HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
- HVX_Vector valueslo = *(HVX_Vector *)word_values_acc;
- HVX_Vector valueshi = *(HVX_Vector *)&word_values_acc[MATRIX_SIZE / 2];
-
- VSCATTER_32_ACC(&vtcm.vscatter32, region_len, offsetslo, valueslo);
- VSCATTER_32_ACC(&vtcm.vscatter32, region_len, offsetshi, valueshi);
+ HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
+ HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
+ HVX_Vector *valueslo = (HVX_Vector *)word_values_acc;
+ HVX_Vector *valueshi = (HVX_Vector *)&word_values_acc[MATRIX_SIZE / 2];
+
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%3 + #0)\n\t"
+ "vscatter(%0, m0, v0.w).w += v1\n\t"
+ : : "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetslo), "r"(valueslo)
+ : "m0", "v0", "v1", "memory");
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%3 + #0)\n\t"
+ "vscatter(%0, m0, v0.w).w += v1\n\t"
+ : : "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetshi), "r"(valueshi)
+ : "m0", "v0", "v1", "memory");
sync_scatter(vtcm.vscatter32);
}
-/* scatter the 32 bit elements using intrinsics */
+/* masked scatter the 32 bit elements using HVX */
void vector_scatter_32_masked(void)
{
- /* copy the offsets and values to vectors */
- HVX_Vector offsetslo = *(HVX_Vector *)word_offsets;
- HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
- HVX_Vector valueslo = *(HVX_Vector *)word_values_masked;
- HVX_Vector valueshi = *(HVX_Vector *)&word_values_masked[MATRIX_SIZE / 2];
- HVX_Vector pred_reglo = *(HVX_Vector *)word_predicates;
- HVX_Vector pred_reghi = *(HVX_Vector *)&word_predicates[MATRIX_SIZE / 2];
- HVX_VectorPred predslo = VAND_VAL(pred_reglo, ~0);
- HVX_VectorPred predshi = VAND_VAL(pred_reghi, ~0);
-
- VSCATTER_32_MASKED(predslo, &vtcm.vscatter32, region_len, offsetslo,
- valueslo);
- VSCATTER_32_MASKED(predshi, &vtcm.vscatter32, region_len, offsetshi,
- valueshi);
+ HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
+ HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
+ HVX_Vector *valueslo = (HVX_Vector *)word_values_masked;
+ HVX_Vector *valueshi = (HVX_Vector *)&word_values_masked[MATRIX_SIZE / 2];
+ HVX_Vector *predslo = (HVX_Vector *)word_predicates;
+ HVX_Vector *predshi = (HVX_Vector *)&word_predicates[MATRIX_SIZE / 2];
+
+ asm ("r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "v1 = vmem(%4 + #0)\n\t"
+ "if (q0) vscatter(%1, m0, v0.w).w = v1\n\t"
+ : : "r"(predslo), "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetslo), "r"(valueslo)
+ : "r1", "q0", "m0", "q0", "v0", "v1", "memory");
+ asm ("r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "v1 = vmem(%4 + #0)\n\t"
+ "if (q0) vscatter(%1, m0, v0.w).w = v1\n\t"
+ : : "r"(predshi), "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetshi), "r"(valueshi)
+ : "r1", "q0", "m0", "q0", "v0", "v1", "memory");
- sync_scatter(vtcm.vscatter16);
+ sync_scatter(vtcm.vscatter32);
}
-/* scatter the 16 bit elements with 32 bit offsets using intrinsics */
+/* scatter the 16 bit elements with 32 bit offsets using HVX */
void vector_scatter_16_32(void)
{
- HVX_VectorPair offsets;
- HVX_Vector values;
-
- /* get the word offsets in a vector pair */
- offsets = *(HVX_VectorPair *)word_offsets;
-
- /* these values need to be shuffled for the scatter */
- values = *(HVX_Vector *)half_values;
- values = VSHUFF_H(values);
-
- VSCATTER_16_32(&vtcm.vscatter16_32, region_len, offsets, values);
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%2 + #1)\n\t"
+ "v2 = vmem(%3 + #0)\n\t"
+ "v2.h = vshuff(v2.h)\n\t" /* shuffle the values for the scatter */
+ "vscatter(%0, m0, v1:0.w).h = v2\n\t"
+ : : "r"(vtcm.vscatter16_32), "r"(region_len),
+ "r"(word_offsets), "r"(half_values)
+ : "m0", "v0", "v1", "v2", "memory");
sync_scatter(vtcm.vscatter16_32);
}
-/* scatter-acc the 16 bit elements with 32 bit offsets using intrinsics */
+/* scatter-accumulate the 16 bit elements with 32 bit offsets using HVX */
void vector_scatter_16_32_acc(void)
{
- HVX_VectorPair offsets;
- HVX_Vector values;
-
- /* get the word offsets in a vector pair */
- offsets = *(HVX_VectorPair *)word_offsets;
-
- /* these values need to be shuffled for the scatter */
- values = *(HVX_Vector *)half_values_acc;
- values = VSHUFF_H(values);
-
- VSCATTER_16_32_ACC(&vtcm.vscatter16_32, region_len, offsets, values);
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%2 + #1)\n\t"
+ "v2 = vmem(%3 + #0)\n\t" \
+ "v2.h = vshuff(v2.h)\n\t" /* shuffle the values for the scatter */
+ "vscatter(%0, m0, v1:0.w).h += v2\n\t"
+ : : "r"(vtcm.vscatter16_32), "r"(region_len),
+ "r"(word_offsets), "r"(half_values_acc)
+ : "m0", "v0", "v1", "v2", "memory");
sync_scatter(vtcm.vscatter16_32);
}
-/* masked scatter the 16 bit elements with 32 bit offsets using intrinsics */
+/* masked scatter the 16 bit elements with 32 bit offsets using HVX */
void vector_scatter_16_32_masked(void)
{
- HVX_VectorPair offsets;
- HVX_Vector values;
- HVX_Vector pred_reg;
-
- /* get the word offsets in a vector pair */
- offsets = *(HVX_VectorPair *)word_offsets;
-
- /* these values need to be shuffled for the scatter */
- values = *(HVX_Vector *)half_values_masked;
- values = VSHUFF_H(values);
-
- pred_reg = *(HVX_Vector *)half_predicates;
- pred_reg = VSHUFF_H(pred_reg);
- HVX_VectorPred preds = VAND_VAL(pred_reg, ~0);
-
- VSCATTER_16_32_MASKED(preds, &vtcm.vscatter16_32, region_len, offsets,
- values);
+ asm ("r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "v0.h = vshuff(v0.h)\n\t" /* shuffle the predicates */
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "v1 = vmem(%3 + #1)\n\t"
+ "v2 = vmem(%4 + #0)\n\t" \
+ "v2.h = vshuff(v2.h)\n\t" /* shuffle the values for the scatter */
+ "if (q0) vscatter(%1, m0, v1:0.w).h = v2\n\t"
+ : : "r"(half_predicates), "r"(vtcm.vscatter16_32), "r"(region_len),
+ "r"(word_offsets), "r"(half_values_masked)
+ : "r1", "q0", "m0", "v0", "v1", "v2", "memory");
sync_scatter(vtcm.vscatter16_32);
}
-/* gather the elements from the scatter16 buffer */
+/* gather the elements from the scatter16 buffer using HVX */
void vector_gather_16(void)
{
- HVX_Vector *vgather = (HVX_Vector *)&vtcm.vgather16;
- HVX_Vector offsets = *(HVX_Vector *)half_offsets;
-
- VGATHER_16(vgather, &vtcm.vscatter16, region_len, offsets);
-
- sync_gather(vgather);
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "{ vtmp.h = vgather(%0, m0, v0.h).h\n\t"
+ " vmem(%3 + #0) = vtmp.new }\n\t"
+ : : "r"(vtcm.vscatter16), "r"(region_len),
+ "r"(half_offsets), "r"(vtcm.vgather16)
+ : "m0", "v0", "memory");
+
+ sync_gather(vtcm.vgather16);
}
static unsigned short gather_16_masked_init(void)
@@ -427,31 +423,51 @@ static unsigned short gather_16_masked_init(void)
return letter | (letter << 8);
}
+/* masked gather the elements from the scatter16 buffer using HVX */
void vector_gather_16_masked(void)
{
- HVX_Vector *vgather = (HVX_Vector *)&vtcm.vgather16;
- HVX_Vector offsets = *(HVX_Vector *)half_offsets;
- HVX_Vector pred_reg = *(HVX_Vector *)half_predicates;
- HVX_VectorPred preds = VAND_VAL(pred_reg, ~0);
-
- *vgather = VSPLAT_H(gather_16_masked_init());
- VGATHER_16_MASKED(vgather, preds, &vtcm.vscatter16, region_len, offsets);
-
- sync_gather(vgather);
+ unsigned short init = gather_16_masked_init();
+
+ asm ("v0.h = vsplat(%5)\n\t"
+ "vmem(%4 + #0) = v0\n\t" /* initialize the write area */
+ "r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "{ if (q0) vtmp.h = vgather(%1, m0, v0.h).h\n\t"
+ " vmem(%4 + #0) = vtmp.new }\n\t"
+ : : "r"(half_predicates), "r"(vtcm.vscatter16), "r"(region_len),
+ "r"(half_offsets), "r"(vtcm.vgather16), "r"(init)
+ : "r1", "q0", "m0", "v0", "memory");
+
+ sync_gather(vtcm.vgather16);
}
-/* gather the elements from the scatter32 buffer */
+/* gather the elements from the scatter32 buffer using HVX */
void vector_gather_32(void)
{
- HVX_Vector *vgatherlo = (HVX_Vector *)&vtcm.vgather32;
- HVX_Vector *vgatherhi =
- (HVX_Vector *)((int)&vtcm.vgather32 + (MATRIX_SIZE * 2));
- HVX_Vector offsetslo = *(HVX_Vector *)word_offsets;
- HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
-
- VGATHER_32(vgatherlo, &vtcm.vscatter32, region_len, offsetslo);
- VGATHER_32(vgatherhi, &vtcm.vscatter32, region_len, offsetshi);
+ HVX_Vector *vgatherlo = (HVX_Vector *)vtcm.vgather32;
+ HVX_Vector *vgatherhi = (HVX_Vector *)&vtcm.vgather32[MATRIX_SIZE / 2];
+ HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
+ HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
+
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "{ vtmp.w = vgather(%0, m0, v0.w).w\n\t"
+ " vmem(%3 + #0) = vtmp.new }\n\t"
+ : : "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetslo), "r"(vgatherlo)
+ : "m0", "v0", "memory");
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "{ vtmp.w = vgather(%0, m0, v0.w).w\n\t"
+ " vmem(%3 + #0) = vtmp.new }\n\t"
+ : : "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetshi), "r"(vgatherhi)
+ : "m0", "v0", "memory");
+ sync_gather(vgatherlo);
sync_gather(vgatherhi);
}
@@ -461,79 +477,88 @@ static unsigned int gather_32_masked_init(void)
return letter | (letter << 8) | (letter << 16) | (letter << 24);
}
+/* masked gather the elements from the scatter32 buffer using HVX */
void vector_gather_32_masked(void)
{
- HVX_Vector *vgatherlo = (HVX_Vector *)&vtcm.vgather32;
- HVX_Vector *vgatherhi =
- (HVX_Vector *)((int)&vtcm.vgather32 + (MATRIX_SIZE * 2));
- HVX_Vector offsetslo = *(HVX_Vector *)word_offsets;
- HVX_Vector offsetshi = *(HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
- HVX_Vector pred_reglo = *(HVX_Vector *)word_predicates;
- HVX_VectorPred predslo = VAND_VAL(pred_reglo, ~0);
- HVX_Vector pred_reghi = *(HVX_Vector *)&word_predicates[MATRIX_SIZE / 2];
- HVX_VectorPred predshi = VAND_VAL(pred_reghi, ~0);
-
- *vgatherlo = VSPLAT_H(gather_32_masked_init());
- *vgatherhi = VSPLAT_H(gather_32_masked_init());
- VGATHER_32_MASKED(vgatherlo, predslo, &vtcm.vscatter32, region_len,
- offsetslo);
- VGATHER_32_MASKED(vgatherhi, predshi, &vtcm.vscatter32, region_len,
- offsetshi);
+ unsigned int init = gather_32_masked_init();
+ HVX_Vector *vgatherlo = (HVX_Vector *)vtcm.vgather32;
+ HVX_Vector *vgatherhi = (HVX_Vector *)&vtcm.vgather32[MATRIX_SIZE / 2];
+ HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
+ HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
+ HVX_Vector *predslo = (HVX_Vector *)word_predicates;
+ HVX_Vector *predshi = (HVX_Vector *)&word_predicates[MATRIX_SIZE / 2];
+
+ asm ("v0.h = vsplat(%5)\n\t"
+ "vmem(%4 + #0) = v0\n\t" /* initialize the write area */
+ "r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "{ if (q0) vtmp.w = vgather(%1, m0, v0.w).w\n\t"
+ " vmem(%4 + #0) = vtmp.new }\n\t"
+ : : "r"(predslo), "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetslo), "r"(vgatherlo), "r"(init)
+ : "r1", "q0", "m0", "v0", "memory");
+ asm ("v0.h = vsplat(%5)\n\t"
+ "vmem(%4 + #0) = v0\n\t" /* initialize the write area */
+ "r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "{ if (q0) vtmp.w = vgather(%1, m0, v0.w).w\n\t"
+ " vmem(%4 + #0) = vtmp.new }\n\t"
+ : : "r"(predshi), "r"(vtcm.vscatter32), "r"(region_len),
+ "r"(offsetshi), "r"(vgatherhi), "r"(init)
+ : "r1", "q0", "m0", "v0", "memory");
sync_gather(vgatherlo);
sync_gather(vgatherhi);
}
-/* gather the elements from the scatter16_32 buffer */
+/* gather the elements from the scatter16_32 buffer using HVX */
void vector_gather_16_32(void)
{
- HVX_Vector *vgather;
- HVX_VectorPair offsets;
- HVX_Vector values;
-
- /* get the vtcm address to gather from */
- vgather = (HVX_Vector *)&vtcm.vgather16_32;
-
- /* get the word offsets in a vector pair */
- offsets = *(HVX_VectorPair *)word_offsets;
-
- VGATHER_16_32(vgather, &vtcm.vscatter16_32, region_len, offsets);
-
- /* deal the elements to get the order back */
- values = *(HVX_Vector *)vgather;
- values = VDEAL_H(values);
-
- /* write it back to vtcm address */
- *(HVX_Vector *)vgather = values;
+ asm ("m0 = %1\n\t"
+ "v0 = vmem(%2 + #0)\n\t"
+ "v1 = vmem(%2 + #1)\n\t"
+ "{ vtmp.h = vgather(%0, m0, v1:0.w).h\n\t"
+ " vmem(%3 + #0) = vtmp.new }\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "v0.h = vdeal(v0.h)\n\t" /* deal the elements to get the order back */
+ "vmem(%3 + #0) = v0\n\t"
+ : : "r"(vtcm.vscatter16_32), "r"(region_len),
+ "r"(word_offsets), "r"(vtcm.vgather16_32)
+ : "m0", "v0", "v1", "memory");
+
+ sync_gather(vtcm.vgather16_32);
}
+/* masked gather the elements from the scatter16_32 buffer using HVX */
void vector_gather_16_32_masked(void)
{
- HVX_Vector *vgather;
- HVX_VectorPair offsets;
- HVX_Vector pred_reg;
- HVX_VectorPred preds;
- HVX_Vector values;
-
- /* get the vtcm address to gather from */
- vgather = (HVX_Vector *)&vtcm.vgather16_32;
-
- /* get the word offsets in a vector pair */
- offsets = *(HVX_VectorPair *)word_offsets;
- pred_reg = *(HVX_Vector *)half_predicates;
- pred_reg = VSHUFF_H(pred_reg);
- preds = VAND_VAL(pred_reg, ~0);
-
- *vgather = VSPLAT_H(gather_16_masked_init());
- VGATHER_16_32_MASKED(vgather, preds, &vtcm.vscatter16_32, region_len,
- offsets);
-
- /* deal the elements to get the order back */
- values = *(HVX_Vector *)vgather;
- values = VDEAL_H(values);
-
- /* write it back to vtcm address */
- *(HVX_Vector *)vgather = values;
+ unsigned short init = gather_16_masked_init();
+
+ asm ("v0.h = vsplat(%5)\n\t"
+ "vmem(%4 + #0) = v0\n\t" /* initialize the write area */
+ "r1 = #-1\n\t"
+ "v0 = vmem(%0 + #0)\n\t"
+ "v0.h = vshuff(v0.h)\n\t" /* shuffle the predicates */
+ "q0 = vand(v0, r1)\n\t"
+ "m0 = %2\n\t"
+ "v0 = vmem(%3 + #0)\n\t"
+ "v1 = vmem(%3 + #1)\n\t"
+ "{ if (q0) vtmp.h = vgather(%1, m0, v1:0.w).h\n\t"
+ " vmem(%4 + #0) = vtmp.new }\n\t"
+ "v0 = vmem(%4 + #0)\n\t"
+ "v0.h = vdeal(v0.h)\n\t" /* deal the elements to get the order back */
+ "vmem(%4 + #0) = v0\n\t"
+ : : "r"(half_predicates), "r"(vtcm.vscatter16_32), "r"(region_len),
+ "r"(word_offsets), "r"(vtcm.vgather16_32), "r"(init)
+ : "r1", "q0", "m0", "v0", "v1", "memory");
+
+ sync_gather(vtcm.vgather16_32);
}
static void check_buffer(const char *name, void *c, void *r, size_t size)
@@ -579,6 +604,7 @@ void scalar_scatter_16_acc(unsigned short *vscatter16)
}
}
+/* scatter-accumulate the 16 bit elements using C */
void check_scatter_16_acc()
{
memset(vscatter16_ref, FILL_CHAR,
@@ -589,7 +615,7 @@ void check_scatter_16_acc()
SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}
-/* scatter the 16 bit elements using C */
+/* masked scatter the 16 bit elements using C */
void scalar_scatter_16_masked(unsigned short *vscatter16)
{
for (int i = 0; i < MATRIX_SIZE; i++) {
@@ -628,7 +654,7 @@ void check_scatter_32()
SCATTER_BUFFER_SIZE * sizeof(unsigned int));
}
-/* scatter the 32 bit elements using C */
+/* scatter-accumulate the 32 bit elements using C */
void scalar_scatter_32_acc(unsigned int *vscatter32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -646,7 +672,7 @@ void check_scatter_32_acc()
SCATTER_BUFFER_SIZE * sizeof(unsigned int));
}
-/* scatter the 32 bit elements using C */
+/* masked scatter the 32 bit elements using C */
void scalar_scatter_32_masked(unsigned int *vscatter32)
{
for (int i = 0; i < MATRIX_SIZE; i++) {
@@ -667,7 +693,7 @@ void check_scatter_32_masked()
SCATTER_BUFFER_SIZE * sizeof(unsigned int));
}
-/* scatter the 32 bit elements using C */
+/* scatter the 16 bit elements with 32 bit offsets using C */
void scalar_scatter_16_32(unsigned short *vscatter16_32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -684,7 +710,7 @@ void check_scatter_16_32()
SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}
-/* scatter the 32 bit elements using C */
+/* scatter-accumulate the 16 bit elements with 32 bit offsets using C */
void scalar_scatter_16_32_acc(unsigned short *vscatter16_32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -702,6 +728,7 @@ void check_scatter_16_32_acc()
SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}
+/* masked scatter the 16 bit elements with 32 bit offsets using C */
void scalar_scatter_16_32_masked(unsigned short *vscatter16_32)
{
for (int i = 0; i < MATRIX_SIZE; i++) {
@@ -738,6 +765,7 @@ void check_gather_16()
MATRIX_SIZE * sizeof(unsigned short));
}
+/* masked gather the elements from the scatter buffer using C */
void scalar_gather_16_masked(unsigned short *vgather16)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -756,7 +784,7 @@ void check_gather_16_masked()
MATRIX_SIZE * sizeof(unsigned short));
}
-/* gather the elements from the scatter buffer using C */
+/* gather the elements from the scatter32 buffer using C */
void scalar_gather_32(unsigned int *vgather32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -772,6 +800,7 @@ void check_gather_32(void)
MATRIX_SIZE * sizeof(unsigned int));
}
+/* masked gather the elements from the scatter32 buffer using C */
void scalar_gather_32_masked(unsigned int *vgather32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -781,7 +810,6 @@ void scalar_gather_32_masked(unsigned int *vgather32)
}
}
-
void check_gather_32_masked(void)
{
memset(vgather32_ref, gather_32_masked_init(),
@@ -791,7 +819,7 @@ void check_gather_32_masked(void)
vgather32_ref, MATRIX_SIZE * sizeof(unsigned int));
}
-/* gather the elements from the scatter buffer using C */
+/* gather the elements from the scatter16_32 buffer using C */
void scalar_gather_16_32(unsigned short *vgather16_32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
@@ -807,6 +835,7 @@ void check_gather_16_32(void)
MATRIX_SIZE * sizeof(unsigned short));
}
+/* masked gather the elements from the scatter16_32 buffer using C */
void scalar_gather_16_32_masked(unsigned short *vgather16_32)
{
for (int i = 0; i < MATRIX_SIZE; ++i) {
--
2.17.1
next prev parent reply other threads:[~2023-01-05 22:33 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-01-05 22:13 [PATCH v3 0/9] Hexagon: COF overrides, new generator, test update Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 1/9] Hexagon (target/hexagon) Add overrides for jumpr31 instructions Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 2/9] Hexagon (target/hexagon) Add overrides for callr Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 3/9] Hexagon (target/hexagon) Add overrides for endloop1/endloop01 Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 4/9] Hexagon (target/hexagon) Add overrides for dealloc-return instructions Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 5/9] Hexagon (target/hexagon) Analyze packet before generating TCG Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 6/9] Hexagon (target/hexagon) Analyze packet for HVX Taylor Simpson
2023-01-05 22:13 ` [PATCH v3 7/9] Hexagon (tests/tcg/hexagon) Update preg_alias.c Taylor Simpson
2023-01-05 22:13 ` Taylor Simpson [this message]
2023-01-05 22:13 ` [PATCH v3 9/9] Hexagon (tests/tcg/hexagon) Enable HVX tests Taylor Simpson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230105221331.12069-9-tsimpson@quicinc.com \
--to=tsimpson@quicinc.com \
--cc=ale@rev.ng \
--cc=anjo@rev.ng \
--cc=bcain@quicinc.com \
--cc=philmd@linaro.org \
--cc=qemu-devel@nongnu.org \
--cc=quic_mathbern@quicinc.com \
--cc=richard.henderson@linaro.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.