[arm]: Improve av1_quantize_fp_neon().
~1.124x to ~1.144x faster than previous version depending on size.
Bug: b/231719821
Change-Id: I4d12ea875b17846f0db00a498081d103390b3aae
diff --git a/av1/encoder/arm/neon/quantize_neon.c b/av1/encoder/arm/neon/quantize_neon.c
index 2f0671a..17dd579 100644
--- a/av1/encoder/arm/neon/quantize_neon.c
+++ b/av1/encoder/arm/neon/quantize_neon.c
@@ -41,6 +41,34 @@
#endif
}
+static INLINE int16x8_t get_max_lane_eob(const int16_t *iscan,
+ int16x8_t v_eobmax,
+ uint16x8_t v_mask) {
+ const int16x8_t v_iscan = vld1q_s16(&iscan[0]);
+ const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, vdupq_n_s16(1));
+ const int16x8_t v_nz_iscan = vbslq_s16(v_mask, v_iscan_plus1, vdupq_n_s16(0));
+ return vmaxq_s16(v_eobmax, v_nz_iscan);
+}
+
+static INLINE uint16x8_t quantize_fp_8(const tran_low_t *coeff_ptr,
+ tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr,
+ int16x8_t v_quant, int16x8_t v_dequant,
+ int16x8_t v_round, int16x8_t v_zero) {
+ const int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]);
+ const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+ const int16x8_t v_abs = vabsq_s16(v_coeff);
+ const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
+ const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
+ const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
+ const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
+ const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
+ const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
+ store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
+ store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
+ return v_nz_mask;
+}
+
void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
const int16_t *zbin_ptr, const int16_t *round_ptr,
const int16_t *quant_ptr,
@@ -56,66 +84,33 @@
// Quantization pass: All coefficients with index >= zero_flag are
// skippable. Note: zero_flag can be zero.
- int i;
const int16x8_t v_zero = vdupq_n_s16(0);
- const int16x8_t v_one = vdupq_n_s16(1);
+ int16x8_t v_quant = vld1q_s16(quant_ptr);
+ int16x8_t v_dequant = vld1q_s16(dequant_ptr);
+ int16x8_t v_round = vld1q_s16(round_ptr);
int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
- int16x8_t v_round = vmovq_n_s16(round_ptr[1]);
- int16x8_t v_quant = vmovq_n_s16(quant_ptr[1]);
- int16x8_t v_dequant = vmovq_n_s16(dequant_ptr[1]);
- // adjust for dc
- v_round = vsetq_lane_s16(round_ptr[0], v_round, 0);
- v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
- v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+ uint16x8_t v_nz_mask;
// process dc and the first seven ac coeffs
- {
- const int16x8_t v_iscan = vld1q_s16(&iscan[0]);
- const int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]);
- const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
- const int16x8_t v_abs = vabsq_s16(v_coeff);
- const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
- const int32x4_t v_tmp_lo =
- vmull_s16(vget_low_s16(v_tmp), vget_low_s16(v_quant));
- const int32x4_t v_tmp_hi =
- vmull_s16(vget_high_s16(v_tmp), vget_high_s16(v_quant));
- const int16x8_t v_tmp2 =
- vcombine_s16(vshrn_n_s32(v_tmp_lo, 16), vshrn_n_s32(v_tmp_hi, 16));
- const uint16x8_t v_nz_mask = vceqq_s16(v_tmp2, v_zero);
- const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, v_one);
- const int16x8_t v_nz_iscan = vbslq_s16(v_nz_mask, v_zero, v_iscan_plus1);
- const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
- const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
- const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
- v_eobmax_76543210 = vmaxq_s16(v_eobmax_76543210, v_nz_iscan);
- store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
- store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
- v_round = vmovq_n_s16(round_ptr[1]);
- v_quant = vmovq_n_s16(quant_ptr[1]);
- v_dequant = vmovq_n_s16(dequant_ptr[1]);
- }
+ v_nz_mask = quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
+ v_dequant, v_round, v_zero);
+ v_eobmax_76543210 = get_max_lane_eob(&iscan[0], v_eobmax_76543210, v_nz_mask);
+ // overwrite the dc constants with ac constants
+ v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+ v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+ v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+
+ count -= 8;
// now process the rest of the ac coeffs
- for (i = 8; i < count; i += 8) {
- const int16x8_t v_iscan = vld1q_s16(&iscan[i]);
- const int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[i]);
- const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
- const int16x8_t v_abs = vabsq_s16(v_coeff);
- const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
- const int32x4_t v_tmp_lo =
- vmull_s16(vget_low_s16(v_tmp), vget_low_s16(v_quant));
- const int32x4_t v_tmp_hi =
- vmull_s16(vget_high_s16(v_tmp), vget_high_s16(v_quant));
- const int16x8_t v_tmp2 =
- vcombine_s16(vshrn_n_s32(v_tmp_lo, 16), vshrn_n_s32(v_tmp_hi, 16));
- const uint16x8_t v_nz_mask = vceqq_s16(v_tmp2, v_zero);
- const int16x8_t v_iscan_plus1 = vaddq_s16(v_iscan, v_one);
- const int16x8_t v_nz_iscan = vbslq_s16(v_nz_mask, v_zero, v_iscan_plus1);
- const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
- const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
- const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
- v_eobmax_76543210 = vmaxq_s16(v_eobmax_76543210, v_nz_iscan);
- store_s16q_to_tran_low(&qcoeff_ptr[i], v_qcoeff);
- store_s16q_to_tran_low(&dqcoeff_ptr[i], v_dqcoeff);
- }
+ do {
+ coeff_ptr += 8;
+ qcoeff_ptr += 8;
+ dqcoeff_ptr += 8;
+ iscan += 8;
+ v_nz_mask = quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
+ v_dequant, v_round, v_zero);
+ v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
+ count -= 8;
+ } while (count > 0);
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
