Optimize Neon implementation of cdef filter functions
Optimize the constraint function, make the computation of max and min
more parallel and simplify the computation of the final result.
Change-Id: I6e1a7993285e74165929dfc8185cb4f397f8478b
diff --git a/av1/common/arm/cdef_block_neon.c b/av1/common/arm/cdef_block_neon.c
index 68a292b..69ea49f 100644
--- a/av1/common/arm/cdef_block_neon.c
+++ b/av1/common/arm/cdef_block_neon.c
@@ -450,36 +450,37 @@
// sign(a-b) * min(abs(a-b), max(0, threshold - (abs(a-b) >> adjdamp)))
static INLINE int16x8_t constrain16(uint16x8_t a, uint16x8_t b,
unsigned int threshold, int adjdamp) {
- int16x8_t diff = vreinterpretq_s16_u16(vsubq_u16(a, b));
- const int16x8_t sign = vshrq_n_s16(diff, 15);
- diff = vabsq_s16(diff);
- const uint16x8_t s =
- vqsubq_u16(vdupq_n_u16(threshold),
- vreinterpretq_u16_s16(vshlq_s16(diff, vdupq_n_s16(-adjdamp))));
- return veorq_s16(vaddq_s16(sign, vminq_s16(diff, vreinterpretq_s16_u16(s))),
- sign);
+ uint16x8_t diff = vabdq_u16(a, b);
+ const uint16x8_t a_gt_b = vcgtq_u16(a, b);
+ const uint16x8_t s = vqsubq_u16(vdupq_n_u16(threshold),
+ vshlq_u16(diff, vdupq_n_s16(-adjdamp)));
+ const int16x8_t clip = vreinterpretq_s16_u16(vminq_u16(diff, s));
+ return vbslq_s16(a_gt_b, clip, vnegq_s16(clip));
}
static INLINE uint16x8_t get_max_primary(const int is_lowbd, uint16x8_t *tap,
uint16x8_t max,
uint16x8_t cdef_large_value_mask) {
if (is_lowbd) {
- uint8x16_t max_u8 = vreinterpretq_u8_u16(tap[0]);
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[1]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[2]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[3]));
- /* The source is 16 bits, however, we only really care about the lower
- 8 bits. The upper 8 bits contain the "large" flag. After the final
- primary max has been calculated, zero out the upper 8 bits. Use this
- to find the "16 bit" max. */
+ // The source is 16 bits, however, we only really care about the lower
+ // 8 bits. The upper 8 bits contain the "large" flag. After the final
+ // primary max has been calculated, zero out the upper 8 bits. Use this
+ // to find the "16 bit" max.
+ uint8x16_t max0 =
+ vmaxq_u8(vreinterpretq_u8_u16(tap[0]), vreinterpretq_u8_u16(tap[1]));
+ uint8x16_t max1 =
+ vmaxq_u8(vreinterpretq_u8_u16(tap[2]), vreinterpretq_u8_u16(tap[3]));
+ max0 = vmaxq_u8(max0, max1);
max = vmaxq_u16(
- max, vandq_u16(vreinterpretq_u16_u8(max_u8), cdef_large_value_mask));
+ max, vandq_u16(vreinterpretq_u16_u8(max0), cdef_large_value_mask));
} else {
- /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
- max = vmaxq_u16(max, vandq_u16(tap[0], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[1], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[2], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[3], cdef_large_value_mask));
+ // Convert CDEF_VERY_LARGE to 0 before calculating max.
+ uint16x8_t max0 = vmaxq_u16(vandq_u16(tap[0], cdef_large_value_mask),
+ vandq_u16(tap[1], cdef_large_value_mask));
+ uint16x8_t max1 = vmaxq_u16(vandq_u16(tap[2], cdef_large_value_mask),
+ vandq_u16(tap[3], cdef_large_value_mask));
+ max0 = vmaxq_u16(max0, max1);
+ max = vmaxq_u16(max0, max);
}
return max;
}
@@ -488,30 +489,37 @@
uint16x8_t max,
uint16x8_t cdef_large_value_mask) {
if (is_lowbd) {
- uint8x16_t max_u8 = vreinterpretq_u8_u16(tap[0]);
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[1]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[2]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[3]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[4]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[5]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[6]));
- max_u8 = vmaxq_u8(max_u8, vreinterpretq_u8_u16(tap[7]));
- /* The source is 16 bits, however, we only really care about the lower
- 8 bits. The upper 8 bits contain the "large" flag. After the final
- primary max has been calculated, zero out the upper 8 bits. Use this
- to find the "16 bit" max. */
+ // The source is 16 bits, however, we only really care about the lower
+ // 8 bits. The upper 8 bits contain the "large" flag. After the final
+ // primary max has been calculated, zero out the upper 8 bits. Use this
+ // to find the "16 bit" max.
+ uint8x16_t max0 =
+ vmaxq_u8(vreinterpretq_u8_u16(tap[0]), vreinterpretq_u8_u16(tap[1]));
+ uint8x16_t max1 =
+ vmaxq_u8(vreinterpretq_u8_u16(tap[2]), vreinterpretq_u8_u16(tap[3]));
+ uint8x16_t max2 =
+ vmaxq_u8(vreinterpretq_u8_u16(tap[4]), vreinterpretq_u8_u16(tap[5]));
+ uint8x16_t max3 =
+ vmaxq_u8(vreinterpretq_u8_u16(tap[6]), vreinterpretq_u8_u16(tap[7]));
+ max0 = vmaxq_u8(max0, max1);
+ max2 = vmaxq_u8(max2, max3);
+ max0 = vmaxq_u8(max0, max2);
max = vmaxq_u16(
- max, vandq_u16(vreinterpretq_u16_u8(max_u8), cdef_large_value_mask));
+ max, vandq_u16(vreinterpretq_u16_u8(max0), cdef_large_value_mask));
} else {
- /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
- max = vmaxq_u16(max, vandq_u16(tap[0], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[1], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[2], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[3], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[4], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[5], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[6], cdef_large_value_mask));
- max = vmaxq_u16(max, vandq_u16(tap[7], cdef_large_value_mask));
+ // Convert CDEF_VERY_LARGE to 0 before calculating max.
+ uint16x8_t max0 = vmaxq_u16(vandq_u16(tap[0], cdef_large_value_mask),
+ vandq_u16(tap[1], cdef_large_value_mask));
+ uint16x8_t max1 = vmaxq_u16(vandq_u16(tap[2], cdef_large_value_mask),
+ vandq_u16(tap[3], cdef_large_value_mask));
+ uint16x8_t max2 = vmaxq_u16(vandq_u16(tap[4], cdef_large_value_mask),
+ vandq_u16(tap[5], cdef_large_value_mask));
+ uint16x8_t max3 = vmaxq_u16(vandq_u16(tap[6], cdef_large_value_mask),
+ vandq_u16(tap[7], cdef_large_value_mask));
+ max0 = vmaxq_u16(max0, max1);
+ max2 = vmaxq_u16(max2, max3);
+ max0 = vmaxq_u16(max0, max2);
+ max = vmaxq_u16(max, max0);
}
return max;
}
@@ -576,10 +584,10 @@
if (clipping_required) {
max = get_max_primary(is_lowbd, tap, max, cdef_large_value_mask);
- min = vminq_u16(min, tap[0]);
- min = vminq_u16(min, tap[1]);
- min = vminq_u16(min, tap[2]);
- min = vminq_u16(min, tap[3]);
+ uint16x8_t min1 = vminq_u16(tap[0], tap[1]);
+ uint16x8_t min2 = vminq_u16(tap[2], tap[3]);
+ min1 = vminq_u16(min1, min2);
+ min = vminq_u16(min, min1);
}
}
@@ -621,22 +629,20 @@
if (clipping_required) {
max = get_max_secondary(is_lowbd, tap, max, cdef_large_value_mask);
- min = vminq_u16(min, tap[0]);
- min = vminq_u16(min, tap[1]);
- min = vminq_u16(min, tap[2]);
- min = vminq_u16(min, tap[3]);
- min = vminq_u16(min, tap[4]);
- min = vminq_u16(min, tap[5]);
- min = vminq_u16(min, tap[6]);
- min = vminq_u16(min, tap[7]);
+ uint16x8_t min0 = vminq_u16(tap[0], tap[1]);
+ uint16x8_t min1 = vminq_u16(tap[2], tap[3]);
+ uint16x8_t min2 = vminq_u16(tap[4], tap[5]);
+ uint16x8_t min3 = vminq_u16(tap[6], tap[7]);
+ min0 = vminq_u16(min0, min1);
+ min2 = vminq_u16(min2, min3);
+ min0 = vminq_u16(min0, min2);
+ min = vminq_u16(min, min0);
}
}
// res = row + ((sum - (sum < 0) + 8) >> 4)
sum = vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
- int16x8_t res = vaddq_s16(sum, vdupq_n_s16(8));
- res = vshrq_n_s16(res, 4);
- res = vaddq_s16(vreinterpretq_s16_u16(s), res);
+ int16x8_t res = vrsraq_n_s16(vreinterpretq_s16_u16(s), sum, 4);
if (clipping_required) {
res = vminq_s16(vmaxq_s16(res, vreinterpretq_s16_u16(min)),
@@ -716,10 +722,10 @@
if (clipping_required) {
max = get_max_primary(is_lowbd, tap, max, cdef_large_value_mask);
- min = vminq_u16(min, tap[0]);
- min = vminq_u16(min, tap[1]);
- min = vminq_u16(min, tap[2]);
- min = vminq_u16(min, tap[3]);
+ uint16x8_t min0 = vminq_u16(tap[0], tap[1]);
+ uint16x8_t min1 = vminq_u16(tap[2], tap[3]);
+ min0 = vminq_u16(min0, min1);
+ min = vminq_u16(min, min0);
}
}
@@ -761,22 +767,21 @@
if (clipping_required) {
max = get_max_secondary(is_lowbd, tap, max, cdef_large_value_mask);
- min = vminq_u16(min, tap[0]);
- min = vminq_u16(min, tap[1]);
- min = vminq_u16(min, tap[2]);
- min = vminq_u16(min, tap[3]);
- min = vminq_u16(min, tap[4]);
- min = vminq_u16(min, tap[5]);
- min = vminq_u16(min, tap[6]);
- min = vminq_u16(min, tap[7]);
+ uint16x8_t min0 = vminq_u16(tap[0], tap[1]);
+ uint16x8_t min1 = vminq_u16(tap[2], tap[3]);
+ uint16x8_t min2 = vminq_u16(tap[4], tap[5]);
+ uint16x8_t min3 = vminq_u16(tap[6], tap[7]);
+ min0 = vminq_u16(min0, min1);
+ min2 = vminq_u16(min2, min3);
+ min0 = vminq_u16(min0, min2);
+ min = vminq_u16(min, min0);
}
}
// res = row + ((sum - (sum < 0) + 8) >> 4)
sum = vaddq_s16(sum, vreinterpretq_s16_u16(vcltq_s16(sum, vdupq_n_s16(0))));
- int16x8_t res = vaddq_s16(sum, vdupq_n_s16(8));
- res = vshrq_n_s16(res, 4);
- res = vaddq_s16(vreinterpretq_s16_u16(s), res);
+ int16x8_t res = vrsraq_n_s16(vreinterpretq_s16_u16(s), sum, 4);
+
if (clipping_required) {
res = vminq_s16(vmaxq_s16(res, vreinterpretq_s16_u16(min)),
vreinterpretq_s16_u16(max));
