Google Git
Sign in
aomedia/aom.git/2e124f462b87996c6d6ddb1fab843c04bdaa14a6^!/.
commit
2e124f462b87996c6d6ddb1fab843c04bdaa14a6
[log]
author
Gerda Zsejke More <gerdazsejke.more@arm.com>
Tue Oct 28 14:51:07 2025 +0100
committer
James Zern <jzern@google.com>
Thu Nov 13 21:14:27 2025 -0800
tree
25dc36754f80dd663a5c5aa9ff35cb1b84a7fde2
parent
1e900cac01098ba2b996dab9683193daec971e2e [diff]
Optimize aom_quantize_b functions

Optimize aom_quantize_b_neon, aom_quantize_b_32x32_neon,
aom_quantize_b_64x64_neon by removing the initial memset function
calls that write zeroes to qcoeff_ptr and dqcoeff_ptr. Instead,
write those zero entries inside the processing loops.

In av1_quantize.c the aom_quantize_b functions are called with NULL
values for qm_ptr and iqm_ptr. The unit tests also have the same
behaviour. When these pointers are not NULL, the
aom_quantize_b_helper_c function is called instead. Remove the NULL
checks for these pointer inside the quantize_b functions.

Change-Id: I0032ac5346992fcb949ecaddf8214e3d0b62efd8

diff --git a/av1/encoder/arm/quantize_neon.c b/av1/encoder/arm/quantize_neon.c
index 8d13e57..e9c0ca7 100644
--- a/av1/encoder/arm/quantize_neon.c
+++ b/av1/encoder/arm/quantize_neon.c

@@ -351,6 +351,33 @@
                               iscan, 2);
 }
 
+static inline uint16x8_t quantize_b_logscale0_8(
+    int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
+    int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
+    tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) {
+  const int16x8_t zero = vdupq_n_s16(0);
+
+  int16x8_t coeff_sign = vreinterpretq_s16_u16(vcltq_s16(coeff, zero));
+
+  int16x8_t tmp = vqaddq_s16(abs, round);
+  tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1);
+  tmp = vqdmulhq_s16(tmp, quant_shift);
+
+  int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
+  qcoeff = vbslq_s16(cond, qcoeff, zero);
+  store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
+
+  int16x8_t dqcoeff = vmulq_s16(tmp, dequant);
+  dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign);
+  dqcoeff = vbslq_s16(cond, dqcoeff, zero);
+  store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff);
+
+  uint16x8_t tmp_mask = vcgtq_s16(tmp, zero);
+  uint16x8_t nz_mask = vandq_u16(tmp_mask, cond);
+
+  return nz_mask;
+}
+
 void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                          const int16_t *zbin_ptr, const int16_t *round_ptr,
                          const int16_t *quant_ptr,
@@ -361,88 +388,58 @@
   (void)quant_shift_ptr;
   (void)scan;
 
-  const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] };
+  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
 
-  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
-  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
+  int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]);
+  int16x8_t v_round = vdupq_n_s16(round_ptr[1]);
+  int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
+  int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
+  // Shift by 1 in order to save one shift in the kernel function.
+  int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1] >> 1);
 
-  const int16x8_t zero = vdupq_n_s16(0);
-  int16x8_t v_eobmax_76543210 = vreinterpretq_s16_u16(vceqq_s16(zero, zero));
-
-  int16x8_t vzbins = vdupq_n_s16(zbins[1]), vround = vdupq_n_s16(round_ptr[1]);
-  int16x8_t vdequant = vdupq_n_s16(dequant_ptr[1]);
-  int16x8_t vquant = vdupq_n_s16(quant_ptr[1]);
-  int16x8_t vquant_shift = vdupq_n_s16(quant_shift_ptr[1]);
-
-  int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]);
-  int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+  int16x8_t v_zbins0 = vsetq_lane_s16(zbin_ptr[0], v_zbins, 0);
+  int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
   int16x8_t v_abs = vabsq_s16(v_coeff);
+  uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
 
-  vzbins = vsetq_lane_s16(zbins[0], vzbins, 0);
-
-  uint16x8_t vcond = vcgeq_s16(v_abs, vzbins);
-  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vcond)), 0);
+  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
   if (nz_check) {
-    vround = vsetq_lane_s16(round_ptr[0], vround, 0);
-    vquant = vsetq_lane_s16(quant_ptr[0], vquant, 0);
-    vdequant = vsetq_lane_s16(dequant_ptr[0], vdequant, 0);
-    vquant_shift = vsetq_lane_s16(quant_shift_ptr[0], vquant_shift, 0);
+    int16x8_t v_round0 = vsetq_lane_s16(round_ptr[0], v_round, 0);
+    int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+    int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+    // Shift by 1 in order to save one shift in the kernel function.
+    int16x8_t v_quant_shift0 =
+        vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0);
 
-    int16x8_t vtmp = vqaddq_s16(v_abs, vround);
-    int16x8_t vtmp2 = vsraq_n_s16(vtmp, vqdmulhq_s16(vtmp, vquant), 1);
-    vtmp2 = vshrq_n_s16(vqdmulhq_s16(vtmp2, vquant_shift), 1);
+    const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
+        v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
+        qcoeff_ptr, dqcoeff_ptr);
 
-    int16x8_t vdest = vsubq_s16(veorq_s16(vtmp2, v_coeff_sign), v_coeff_sign);
-    int16x8_t coeff_nz_mask =
-        vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&qcoeff_ptr[0]));
-    store_s16q_to_tran_low(&qcoeff_ptr[0], coeff_nz_mask);
-    int16x8_t v_deq_abs = vmulq_s16(vtmp2, vdequant);
-
-    vdest = vsubq_s16(veorq_s16(v_deq_abs, v_coeff_sign), v_coeff_sign);
-    coeff_nz_mask =
-        vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&dqcoeff_ptr[0]));
-    store_s16q_to_tran_low(&dqcoeff_ptr[0], coeff_nz_mask);
-
-    vround = vsetq_lane_s16(round_ptr[1], vround, 0);
-    vquant = vsetq_lane_s16(quant_ptr[1], vquant, 0);
-    vdequant = vsetq_lane_s16(dequant_ptr[1], vdequant, 0);
-    vquant_shift = vsetq_lane_s16(quant_shift_ptr[1], vquant_shift, 0);
-
-    uint16x8_t vtmp_mask = vcgtq_s16(vtmp2, zero);
-    const uint16x8_t v_nz_mask = vandq_u16(vtmp_mask, vcond);
-    int16x8_t v_iscan = vld1q_s16(&iscan[0]);
-    vcond = vandq_u16(v_nz_mask, vcgtq_s16(v_iscan, v_eobmax_76543210));
-    v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210);
+    int16x8_t v_iscan = vld1q_s16(iscan);
+    int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
+    v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+  } else {
+    store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
+    store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
   }
-  vzbins = vsetq_lane_s16(zbins[1], vzbins, 0);
 
   for (int i = 8; i < n_coeffs; i += 8) {
-    v_coeff = load_tran_low_to_s16q(&coeff_ptr[i]);
-    v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+    v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
     v_abs = vabsq_s16(v_coeff);
-    vcond = vcgeq_s16(v_abs, vzbins);
+    v_cond = vcgeq_s16(v_abs, v_zbins);
 
-    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vcond)), 0);
+    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
     if (nz_check) {
-      int16x8_t vtmp = vqaddq_s16(v_abs, vround);
-      int16x8_t vtmp2 = vsraq_n_s16(vtmp, vqdmulhq_s16(vtmp, vquant), 1);
+      const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
+          v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
+          qcoeff_ptr + i, dqcoeff_ptr + i);
 
-      vtmp2 = vshrq_n_s16(vqdmulhq_s16(vtmp2, vquant_shift), 1);
-      int16x8_t vdest = vsubq_s16(veorq_s16(vtmp2, v_coeff_sign), v_coeff_sign);
-      int16x8_t coeff_nz_mask =
-          vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&qcoeff_ptr[i]));
-      store_s16q_to_tran_low(&qcoeff_ptr[i], coeff_nz_mask);
-      int16x8_t v_deq_abs = vmulq_s16(vtmp2, vdequant);
-      vdest = vsubq_s16(veorq_s16(v_deq_abs, v_coeff_sign), v_coeff_sign);
-      coeff_nz_mask =
-          vbslq_s16(vcond, vdest, load_tran_low_to_s16q(&dqcoeff_ptr[i]));
-      store_s16q_to_tran_low(&dqcoeff_ptr[i], coeff_nz_mask);
-
-      uint16x8_t vtmp_mask = vcgtq_s16(vtmp2, zero);
-      const uint16x8_t v_nz_mask = vandq_u16(vtmp_mask, vcond);
-      int16x8_t v_iscan = vld1q_s16(&iscan[i]);
-      vcond = vandq_u16(v_nz_mask, vcgtq_s16(v_iscan, v_eobmax_76543210));
-      v_eobmax_76543210 = vbslq_s16(vcond, v_iscan, v_eobmax_76543210);
+      int16x8_t v_iscan = vld1q_s16(iscan + i);
+      int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
+      v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+    } else {
+      store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
+      store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
     }
   }
   *eob_ptr = get_max_eob(v_eobmax_76543210);
@@ -899,6 +896,35 @@
   }
 }
 
+static inline uint16x8_t quantize_b_logscale1_8(
+    int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
+    int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
+    tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) {
+  const int16x8_t zero = vdupq_n_s16(0);
+
+  int16x8_t coeff_sign = vreinterpretq_s16_u16(vcltq_s16(coeff, zero));
+
+  int16x8_t tmp = vqaddq_s16(abs, round);
+  tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1);
+  tmp = vqdmulhq_s16(tmp, quant_shift);
+
+  int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
+  qcoeff = vbslq_s16(cond, qcoeff, zero);
+  store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
+
+  // Shift by log_scale = 1.
+  int16x8_t dqcoeff = vreinterpretq_s16_u16(vhaddq_u16(
+      vreinterpretq_u16_s16(vmulq_s16(tmp, dequant)), vdupq_n_u16(0)));
+  dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign);
+  dqcoeff = vbslq_s16(cond, dqcoeff, zero);
+  store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff);
+
+  uint16x8_t tmp_mask = vcgtq_s16(tmp, zero);
+  const uint16x8_t nz_mask = vandq_u16(tmp_mask, cond);
+
+  return nz_mask;
+}
+
 void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                                const int16_t *zbin_ptr,
                                const int16_t *round_ptr,
@@ -907,10 +933,100 @@
                                tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
                                const int16_t *dequant_ptr, uint16_t *eob_ptr,
                                const int16_t *scan, const int16_t *iscan) {
-  aom_quantize_b_helper_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
-                             quant_ptr, quant_shift_ptr, qcoeff_ptr,
-                             dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
-                             NULL, NULL, 1);
+  (void)scan;
+
+  const int log_scale = 1;
+  const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
+                         ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
+  const int rounds[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
+                          ROUND_POWER_OF_TWO(round_ptr[1], log_scale) };
+
+  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
+
+  int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
+  int16x8_t v_round = vdupq_n_s16(rounds[1]);
+  int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
+  int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
+  int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
+
+  int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
+  int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
+  int16x8_t v_abs = vabsq_s16(v_coeff);
+  uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
+
+  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
+  if (nz_check) {
+    int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0);
+    int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+    int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+    int16x8_t v_quant_shift0 =
+        vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+
+    const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
+        v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
+        qcoeff_ptr, dqcoeff_ptr);
+
+    int16x8_t v_iscan = vld1q_s16(iscan);
+    int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
+    v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+  } else {
+    store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
+    store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+  }
+
+  for (int i = 8; i < n_coeffs; i += 8) {
+    v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
+    v_abs = vabsq_s16(v_coeff);
+    v_cond = vcgeq_s16(v_abs, v_zbins);
+
+    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
+    if (nz_check) {
+      const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
+          v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
+          qcoeff_ptr + i, dqcoeff_ptr + i);
+
+      int16x8_t v_iscan = vld1q_s16(iscan + i);
+      int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
+      v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+    } else {
+      store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
+      store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+    }
+  }
+  *eob_ptr = get_max_eob(v_eobmax_76543210);
+}
+
+static inline uint16x8_t quantize_b_logscale2_8(
+    int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
+    int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
+    tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr) {
+  const int16x8_t zero = vdupq_n_s16(0);
+  const int16x8_t one = vdupq_n_s16(1);
+
+  int16x8_t coeff_sign = vreinterpretq_s16_u16(vcltq_s16(coeff, zero));
+
+  int16x8_t tmp = vqaddq_s16(abs, round);
+  tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1);
+  int16x8_t ones = vandq_s16(vshrq_n_s16(vmulq_s16(tmp, quant_shift), 14), one);
+  tmp = vqdmulhq_s16(tmp, quant_shift);
+  tmp = vaddq_s16(vshlq_s16(tmp, one), ones);
+
+  int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
+  qcoeff = vbslq_s16(cond, qcoeff, zero);
+  store_s16q_to_tran_low(qcoeff_ptr, qcoeff);
+
+  // Shift right by log_scale = 2.
+  int16x8_t dqcoeff = vreinterpretq_s16_u16(
+      vshrq_n_u16(vreinterpretq_u16_s16(vmulq_s16(tmp, dequant)), 2));
+  dqcoeff = vorrq_s16(vshlq_n_s16(vqdmulhq_s16(tmp, dequant), 13), dqcoeff);
+  dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign);
+  dqcoeff = vbslq_s16(cond, dqcoeff, zero);
+  store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff);
+
+  uint16x8_t tmp_mask = vcgtq_s16(tmp, zero);
+  const uint16x8_t nz_mask = vandq_u16(tmp_mask, cond);
+
+  return nz_mask;
 }
 
 void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
@@ -921,8 +1037,65 @@
                                tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
                                const int16_t *dequant_ptr, uint16_t *eob_ptr,
                                const int16_t *scan, const int16_t *iscan) {
-  aom_quantize_b_helper_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
-                             quant_ptr, quant_shift_ptr, qcoeff_ptr,
-                             dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
-                             NULL, NULL, 2);
+  (void)scan;
+
+  const int log_scale = 2;
+  const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
+                         ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
+  const int rounds[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
+                          ROUND_POWER_OF_TWO(round_ptr[1], log_scale) };
+
+  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
+
+  int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
+  int16x8_t v_round = vdupq_n_s16(rounds[1]);
+  int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
+  int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
+  int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
+
+  int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
+  int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
+  int16x8_t v_abs = vabsq_s16(v_coeff);
+  uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
+
+  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
+  if (nz_check) {
+    int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0);
+    int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+    int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+    int16x8_t v_quant_shift0 =
+        vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+
+    const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
+        v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
+        qcoeff_ptr, dqcoeff_ptr);
+
+    int16x8_t v_iscan = vld1q_s16(iscan);
+    int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
+    v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+  } else {
+    store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
+    store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+  }
+
+  for (int i = 8; i < n_coeffs; i += 8) {
+    v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
+    v_abs = vabsq_s16(v_coeff);
+    v_cond = vcgeq_s16(v_abs, v_zbins);
+
+    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
+    if (nz_check) {
+      const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
+          v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
+          qcoeff_ptr + i, dqcoeff_ptr + i);
+
+      int16x8_t v_iscan = vld1q_s16(iscan + i);
+      int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
+      v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+    } else {
+      store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
+      store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+    }
+  }
+  *eob_ptr = get_max_eob(v_eobmax_76543210);
 }