Downsample sad computation in hd motion search
This CL adds a speed feature to downsample the SAD computation during
motion search by skipping every other row in sad computation.
The downsampling is only performed when there are sufficiently many rows
to get a good estimation. Empirically, this happens when there are at
leas 16 rows.
Currently this speed feature is only enabled on hdres set.
Performance:
SPD_SET | OVR_PSNR | AVG_PSNR | SSIM | SPD
0 | +0.021% | +0.025% | +0.084% | +2.3%
1 | +0.053% | +0.071% | +0.136% | +3.8%
2 | +0.037% | +0.055% | +0.091% | +4.3%
3 | +0.025% | +0.038% | +0.074% | +2.6%
4 | +0.029% | +0.043% | +0.093% | +3.0%
5 | +0.117% | +0.128% | +0.194% | +3.3%
6 | +0.065% | +0.082% | +0.179% | +3.2%
BUG=aomedia:2781
STATS_CHANGED
Change-Id: Ibf2afd9a7ffed939897249527b41bbaa4152a62c
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 5028929..b186294 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -635,6 +635,7 @@
foreach (@block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_sad${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
+ add_proto qw/unsigned int/, "aom_sad_skip_${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
add_proto qw/unsigned int/, "aom_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
add_proto qw/unsigned int/, "aom_dist_wtd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred, const DIST_WTD_COMP_PARAMS *jcp_param";
}
@@ -656,6 +657,34 @@
specialize qw/aom_sad4x8 msa sse2/;
specialize qw/aom_sad4x4 neon msa sse2/;
+ specialize qw/aom_sad4x16 sse2/;
+ specialize qw/aom_sad16x4 sse2/;
+ specialize qw/aom_sad8x32 sse2/;
+ specialize qw/aom_sad32x8 sse2/;
+ specialize qw/aom_sad16x64 sse2/;
+ specialize qw/aom_sad64x16 sse2/;
+
+ specialize qw/aom_sad_skip_128x128 avx2 sse2/;
+ specialize qw/aom_sad_skip_128x64 avx2 sse2/;
+ specialize qw/aom_sad_skip_64x128 avx2 sse2/;
+ specialize qw/aom_sad_skip_64x64 avx2 sse2/;
+ specialize qw/aom_sad_skip_64x32 avx2 sse2/;
+ specialize qw/aom_sad_skip_32x64 avx2 sse2/;
+ specialize qw/aom_sad_skip_32x32 avx2 sse2/;
+ specialize qw/aom_sad_skip_32x16 avx2 sse2/;
+ specialize qw/aom_sad_skip_16x32 sse2/;
+ specialize qw/aom_sad_skip_16x16 sse2/;
+ specialize qw/aom_sad_skip_16x8 sse2/;
+ specialize qw/aom_sad_skip_8x16 sse2/;
+ specialize qw/aom_sad_skip_8x8 sse2/;
+ specialize qw/aom_sad_skip_4x8 sse2/;
+
+ specialize qw/aom_sad_skip_4x16 sse2/;
+ specialize qw/aom_sad_skip_8x32 sse2/;
+ specialize qw/aom_sad_skip_32x8 sse2/;
+ specialize qw/aom_sad_skip_16x64 sse2/;
+ specialize qw/aom_sad_skip_64x16 sse2/;
+
specialize qw/aom_sad128x128_avg avx2 sse2/;
specialize qw/aom_sad128x64_avg avx2 sse2/;
specialize qw/aom_sad64x128_avg avx2 sse2/;
@@ -673,19 +702,12 @@
specialize qw/aom_sad4x8_avg msa sse2/;
specialize qw/aom_sad4x4_avg msa sse2/;
- specialize qw/aom_sad4x16 sse2/;
- specialize qw/aom_sad16x4 sse2/;
- specialize qw/aom_sad8x32 sse2/;
- specialize qw/aom_sad32x8 sse2/;
- specialize qw/aom_sad16x64 sse2/;
- specialize qw/aom_sad64x16 sse2/;
-
- specialize qw/aom_sad4x16_avg sse2/;
- specialize qw/aom_sad16x4_avg sse2/;
- specialize qw/aom_sad8x32_avg sse2/;
- specialize qw/aom_sad32x8_avg sse2/;
- specialize qw/aom_sad16x64_avg sse2/;
- specialize qw/aom_sad64x16_avg sse2/;
+ specialize qw/aom_sad4x16_avg sse2/;
+ specialize qw/aom_sad16x4_avg sse2/;
+ specialize qw/aom_sad8x32_avg sse2/;
+ specialize qw/aom_sad32x8_avg sse2/;
+ specialize qw/aom_sad16x64_avg sse2/;
+ specialize qw/aom_sad64x16_avg sse2/;
specialize qw/aom_dist_wtd_sad128x128_avg ssse3/;
specialize qw/aom_dist_wtd_sad128x64_avg ssse3/;
@@ -729,6 +751,7 @@
foreach (@block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "aom_highbd_sad${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
+ add_proto qw/unsigned int/, "aom_highbd_sad_skip_${w}x${h}", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride";
add_proto qw/unsigned int/, "aom_highbd_sad${w}x${h}_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred";
if ($w != 128 && $h != 128 && $w != 4) {
specialize "aom_highbd_sad${w}x${h}", qw/sse2/;
@@ -747,9 +770,39 @@
specialize qw/aom_highbd_sad16x32 avx2 sse2/;
specialize qw/aom_highbd_sad16x16 avx2 sse2/;
specialize qw/aom_highbd_sad16x8 avx2 sse2/;
- specialize qw/aom_highbd_sad8x4 sse2/;
- specialize qw/aom_highbd_sad4x8 sse2/;
- specialize qw/aom_highbd_sad4x4 sse2/;
+ specialize qw/aom_highbd_sad8x16 sse2/;
+ specialize qw/aom_highbd_sad8x8 sse2/;
+ specialize qw/aom_highbd_sad8x4 sse2/;
+ specialize qw/aom_highbd_sad4x8 sse2/;
+ specialize qw/aom_highbd_sad4x4 sse2/;
+
+ specialize qw/aom_highbd_sad4x16 sse2/;
+ specialize qw/aom_highbd_sad16x4 avx2 sse2/;
+ specialize qw/aom_highbd_sad8x32 sse2/;
+ specialize qw/aom_highbd_sad32x8 avx2 sse2/;
+ specialize qw/aom_highbd_sad16x64 avx2 sse2/;
+ specialize qw/aom_highbd_sad64x16 avx2 sse2/;
+
+ specialize qw/aom_highbd_sad_skip_128x128 avx2/;
+ specialize qw/aom_highbd_sad_skip_128x64 avx2/;
+ specialize qw/aom_highbd_sad_skip_64x128 avx2/;
+ specialize qw/aom_highbd_sad_skip_64x64 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_64x32 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x64 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x32 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x16 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x32 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x16 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x8 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_8x16 sse2/;
+ specialize qw/aom_highbd_sad_skip_8x8 sse2/;
+ specialize qw/aom_highbd_sad_skip_4x8 sse2/;
+
+ specialize qw/aom_highbd_sad_skip_4x16 sse2/;
+ specialize qw/aom_highbd_sad_skip_8x32 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x8 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x64 avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_64x16 avx2 sse2/;
specialize qw/aom_highbd_sad128x128_avg avx2/;
specialize qw/aom_highbd_sad128x64_avg avx2/;
@@ -766,13 +819,6 @@
specialize qw/aom_highbd_sad4x8_avg sse2/;
specialize qw/aom_highbd_sad4x4_avg sse2/;
- specialize qw/aom_highbd_sad4x16 sse2/;
- specialize qw/aom_highbd_sad16x4 avx2 sse2/;
- specialize qw/aom_highbd_sad8x32 sse2/;
- specialize qw/aom_highbd_sad32x8 avx2 sse2/;
- specialize qw/aom_highbd_sad16x64 avx2 sse2/;
- specialize qw/aom_highbd_sad64x16 avx2 sse2/;
-
specialize qw/aom_highbd_sad4x16_avg sse2/;
specialize qw/aom_highbd_sad16x4_avg avx2 sse2/;
specialize qw/aom_highbd_sad8x32_avg sse2/;
@@ -825,6 +871,7 @@
($w, $h) = @$_;
add_proto qw/void/, "aom_sad${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
add_proto qw/void/, "aom_sad${w}x${h}x4d_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, const uint8_t *second_pred, uint32_t *sad_array";
+ add_proto qw/void/, "aom_sad_skip_${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
add_proto qw/void/, "aom_masked_sad${w}x${h}x4d", "const uint8_t *src, int src_stride, const uint8_t *ref[], int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask, unsigned sads[]";
}
@@ -857,6 +904,31 @@
specialize qw/aom_sad32x8x4d sse2/;
specialize qw/aom_sad64x16x4d sse2/;
+ specialize qw/aom_sad_skip_128x128x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_128x64x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_64x128x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_64x64x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_64x32x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_64x16x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_32x64x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_32x32x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_32x16x4d avx2 sse2/;
+ specialize qw/aom_sad_skip_32x8x4d avx2 sse2/;
+
+ specialize qw/aom_sad_skip_16x64x4d sse2/;
+ specialize qw/aom_sad_skip_16x32x4d sse2/;
+ specialize qw/aom_sad_skip_16x16x4d sse2/;
+ specialize qw/aom_sad_skip_16x8x4d sse2/;
+ specialize qw/aom_sad_skip_8x16x4d sse2/;
+ specialize qw/aom_sad_skip_8x8x4d sse2/;
+ specialize qw/aom_sad_skip_4x16x4d sse2/;
+ specialize qw/aom_sad_skip_4x8x4d sse2/;
+ specialize qw/aom_sad_skip_4x32x4d sse2/;
+ specialize qw/aom_sad_skip_4x16x4d sse2/;
+ specialize qw/aom_sad_skip_8x32x4d sse2/;
+ specialize qw/aom_sad_skip_32x8x4d sse2/;
+ specialize qw/aom_sad_skip_64x16x4d sse2/;
+
specialize qw/aom_sad128x128x4d_avg sse2/;
specialize qw/aom_sad128x64x4d_avg sse2/;
specialize qw/aom_sad64x128x4d_avg sse2/;
@@ -921,6 +993,7 @@
foreach (@block_sizes) {
($w, $h) = @$_;
add_proto qw/void/, "aom_highbd_sad${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
+ add_proto qw/void/, "aom_highbd_sad_skip_${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[], int ref_stride, uint32_t *sad_array";
if ($w != 128 && $h != 128) {
specialize "aom_highbd_sad${w}x${h}x4d", qw/sse2/;
}
@@ -948,6 +1021,27 @@
specialize qw/aom_highbd_sad32x8x4d avx2 sse2/;
specialize qw/aom_highbd_sad16x64x4d avx2 sse2/;
specialize qw/aom_highbd_sad64x16x4d avx2 sse2/;
+
+ specialize qw/aom_highbd_sad_skip_128x128x4d avx2/;
+ specialize qw/aom_highbd_sad_skip_128x64x4d avx2/;
+ specialize qw/aom_highbd_sad_skip_64x128x4d avx2/;
+ specialize qw/aom_highbd_sad_skip_64x64x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_64x32x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x64x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x32x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_32x16x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x32x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x16x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x8x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_8x16x4d sse2/;
+ specialize qw/aom_highbd_sad_skip_8x8x4d sse2/;
+ specialize qw/aom_highbd_sad_skip_4x8x4d sse2/;
+
+ specialize qw/aom_highbd_sad_skip_4x16x4d sse2/;
+ specialize qw/aom_highbd_sad_skip_8x32x4d sse2/;
+ specialize qw/aom_highbd_sad_skip_32x8x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_16x64x4d avx2 sse2/;
+ specialize qw/aom_highbd_sad_skip_64x16x4d avx2 sse2/;
}
#
# Avg
diff --git a/aom_dsp/sad.c b/aom_dsp/sad.c
index 8ddc683..d72b6c6 100644
--- a/aom_dsp/sad.c
+++ b/aom_dsp/sad.c
@@ -61,6 +61,11 @@
aom_dist_wtd_comp_avg_pred_c(comp_pred, second_pred, m, n, ref, \
ref_stride, jcp_param); \
return sad(src, src_stride, comp_pred, m, m, n); \
+ } \
+ unsigned int aom_sad_skip_##m##x##n##_c(const uint8_t *src, int src_stride, \
+ const uint8_t *ref, \
+ int ref_stride) { \
+ return 2 * sad(src, 2 * src_stride, ref, 2 * ref_stride, (m), (n / 2)); \
}
// Calculate sad against 4 reference locations and store each in sad_array
@@ -82,6 +87,15 @@
sad_array[i] = aom_sad##m##x##n##_avg_c(src, src_stride, ref_array[i], \
ref_stride, second_pred); \
} \
+ } \
+ void aom_sad_skip_##m##x##n##x4d_c(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref_array[], \
+ int ref_stride, uint32_t *sad_array) { \
+ int i; \
+ for (i = 0; i < 4; ++i) { \
+ sad_array[i] = 2 * sad(src, 2 * src_stride, ref_array[i], \
+ 2 * ref_stride, (m), (n / 2)); \
+ } \
}
// 128x128
@@ -227,6 +241,12 @@
aom_highbd_dist_wtd_comp_avg_pred(comp_pred8, second_pred, m, n, ref, \
ref_stride, jcp_param); \
return highbd_sadb(src, src_stride, comp_pred8, m, m, n); \
+ } \
+ unsigned int aom_highbd_sad_skip_##m##x##n##_c( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, \
+ int ref_stride) { \
+ return 2 * \
+ highbd_sad(src, 2 * src_stride, ref, 2 * ref_stride, (m), (n / 2)); \
}
#define highbd_sadMxNx4D(m, n) \
@@ -238,6 +258,15 @@
sad_array[i] = aom_highbd_sad##m##x##n##_c(src, src_stride, \
ref_array[i], ref_stride); \
} \
+ } \
+ void aom_highbd_sad_skip_##m##x##n##x4d_c( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[], \
+ int ref_stride, uint32_t *sad_array) { \
+ int i; \
+ for (i = 0; i < 4; ++i) { \
+ sad_array[i] = 2 * highbd_sad(src, 2 * src_stride, ref_array[i], \
+ 2 * ref_stride, (m), (n / 2)); \
+ } \
}
// 128x128
diff --git a/aom_dsp/variance.h b/aom_dsp/variance.h
index 4550c17..428afd0 100644
--- a/aom_dsp/variance.h
+++ b/aom_dsp/variance.h
@@ -90,11 +90,15 @@
typedef struct aom_variance_vtable {
aom_sad_fn_t sdf;
+ // Same as normal sad, but downsample the rows by a factor of 2.
+ aom_sad_fn_t sdsf;
aom_sad_avg_fn_t sdaf;
aom_variance_fn_t vf;
aom_subpixvariance_fn_t svf;
aom_subp_avg_variance_fn_t svaf;
aom_sad_multi_d_fn_t sdx4df;
+ // Same as sadx4, but downsample the rows by a factor of 2.
+ aom_sad_multi_d_fn_t sdsx4df;
aom_masked_sad_fn_t msdf;
aom_masked_subpixvariance_fn_t msvf;
aom_obmc_sad_fn_t osdf;
diff --git a/aom_dsp/x86/highbd_sad4d_sse2.asm b/aom_dsp/x86/highbd_sad4d_sse2.asm
index e0d2252..7ae1ca1 100644
--- a/aom_dsp/x86/highbd_sad4d_sse2.asm
+++ b/aom_dsp/x86/highbd_sad4d_sse2.asm
@@ -215,15 +215,28 @@
; void aom_highbd_sadNxNx4d_sse2(uint8_t *src, int src_stride,
; uint8_t *ref[4], int ref_stride,
; uint32_t res[4]);
-; where NxN = 64x64, 32x32, 16x16, 16x8, 8x16 or 8x8
-%macro HIGH_SADNXN4D 2
+; Macro Arguments:
+; 1: Width
+; 2: Height
+; 3: If 0, then normal sad, if 2, then skip every other row
+%macro HIGH_SADNXN4D 2-3 0
+%if %3 == 0 ; normal sad
%if UNIX64
cglobal highbd_sad%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \
res, ref2, ref3, ref4
%else
cglobal highbd_sad%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, \
ref2, ref3, ref4
-%endif
+%endif ; UNIX64
+%else ; %3 == 2, downsample
+%if UNIX64
+cglobal highbd_sad_skip_%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \
+ res, ref2, ref3, ref4
+%else
+cglobal highbd_sad_skip_%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, \
+ ref2, ref3, ref4
+%endif ; UNIX64
+%endif ; sad/avg/skip
; set m1
push srcq
@@ -232,6 +245,10 @@
pshufd m1, m1, 0x0
pop srcq
+%if %3 == 2 ; skip rows
+ lea src_strided, [2*src_strided]
+ lea ref_strided, [2*ref_strided]
+%endif ; skip rows
movsxdifnidn src_strideq, src_strided
movsxdifnidn ref_strideq, ref_strided
mov ref2q, [ref1q+gprsize*1]
@@ -247,9 +264,15 @@
shl ref1q, 1
HIGH_PROCESS_%1x2x4 1, 0, 0, src_strideq, ref_strideq, 1
-%rep (%2-4)/2
+%if %3 == 2 ; Downsampling by two
+%define num_rep (%2-8)/4
+%else
+%define num_rep (%2-4)/2
+%endif
+%rep num_rep
HIGH_PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 1
%endrep
+%undef rep
HIGH_PROCESS_%1x2x4 0, 0, 0, src_strideq, ref_strideq, 0
; N.B. HIGH_PROCESS outputs dwords (32 bits)
; so in high bit depth even the smallest width (4) needs 128bits i.e. XMM
@@ -268,6 +291,9 @@
paddd m4, m0
paddd m6, m1
punpcklqdq m4, m6
+%if %3 == 2 ; skip rows
+ pslld m4, 1
+%endif
movifnidn r4, r4mp
movu [r4], m4
RET
@@ -294,3 +320,25 @@
HIGH_SADNXN4D 32, 8
HIGH_SADNXN4D 16, 64
HIGH_SADNXN4D 64, 16
+
+HIGH_SADNXN4D 64, 64, 2
+HIGH_SADNXN4D 64, 32, 2
+HIGH_SADNXN4D 32, 64, 2
+HIGH_SADNXN4D 32, 32, 2
+HIGH_SADNXN4D 32, 16, 2
+HIGH_SADNXN4D 16, 32, 2
+HIGH_SADNXN4D 16, 16, 2
+HIGH_SADNXN4D 16, 8, 2
+HIGH_SADNXN4D 8, 16, 2
+HIGH_SADNXN4D 8, 8, 2
+HIGH_SADNXN4D 4, 8, 2
+HIGH_SADNXN4D 4, 16, 2
+HIGH_SADNXN4D 8, 32, 2
+HIGH_SADNXN4D 32, 8, 2
+HIGH_SADNXN4D 16, 64, 2
+HIGH_SADNXN4D 64, 16, 2
+
+; Current code cannot handle the case when the height is downsampled to 2
+; HIGH_SADNXN4D 16, 4, 2
+; HIGH_SADNXN4D 8, 4, 2
+; HIGH_SADNXN4D 4, 4, 2
diff --git a/aom_dsp/x86/highbd_sad_sse2.asm b/aom_dsp/x86/highbd_sad_sse2.asm
index 09e64d5..58f1ac9 100644
--- a/aom_dsp/x86/highbd_sad_sse2.asm
+++ b/aom_dsp/x86/highbd_sad_sse2.asm
@@ -15,6 +15,11 @@
SECTION .text
+; Macro Arguments
+; Arg 1: Width
+; Arg 2: Height
+; Arg 3: Number of general purpose registers: 5 for 32-bit build, 6 for 64-bit
+; Arg 4: Type of function: if 0, normal sad; if 1, avg; if 2, skip rows
%macro HIGH_SAD_FN 4
%if %4 == 0
%if %3 == 5
@@ -23,7 +28,7 @@
cglobal highbd_sad%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, \
src_stride3, ref_stride3, n_rows
%endif ; %3 == 5/7
-%else ; avg
+%elif %4 == 1 ; avg
%if %3 == 5
cglobal highbd_sad%1x%2_avg, 5, 1 + %3, 7, src, src_stride, ref, ref_stride, \
second_pred, n_rows
@@ -38,7 +43,18 @@
%define n_rowsd dword r0m
%endif ; x86-32/64
%endif ; %3 == 5/7
-%endif ; avg/sad
+%else ; %4 == 2, skip rows
+%if %3 == 5
+cglobal highbd_sad_skip_%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, n_rows
+%else ; %3 == 7
+cglobal highbd_sad_skip_%1x%2, 4, %3, 7, src, src_stride, ref, ref_stride, \
+ src_stride3, ref_stride3, n_rows
+%endif ; %3 == 5/7
+%endif ; sad/avg/skip
+%if %4 == 2 ; double the stride if we are skipping rows
+ lea src_strided, [src_strided*2]
+ lea ref_strided, [ref_strided*2]
+%endif
movsxdifnidn src_strideq, src_strided
movsxdifnidn ref_strideq, ref_strided
%if %3 == 7
@@ -57,7 +73,11 @@
; uint8_t *ref, int ref_stride);
%macro HIGH_SAD64XN 1-2 0
HIGH_SAD_FN 64, %1, 5, %2
+%if %2 == 2 ; skip rows, so divide number of rows by 2
+ mov n_rowsd, %1/2
+%else
mov n_rowsd, %1
+%endif
pxor m0, m0
pxor m6, m6
@@ -149,6 +169,9 @@
punpckldq m0, m6
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
@@ -156,16 +179,23 @@
INIT_XMM sse2
HIGH_SAD64XN 64 ; highbd_sad64x64_sse2
HIGH_SAD64XN 32 ; highbd_sad64x32_sse2
+HIGH_SAD64XN 16 ; highbd_sad_64x16_sse2
HIGH_SAD64XN 64, 1 ; highbd_sad64x64_avg_sse2
HIGH_SAD64XN 32, 1 ; highbd_sad64x32_avg_sse2
-HIGH_SAD64XN 16 ; highbd_sad_64x16_sse2
HIGH_SAD64XN 16, 1 ; highbd_sad_64x16_avg_sse2
+HIGH_SAD64XN 64, 2 ; highbd_sad_skip_64x64_sse2
+HIGH_SAD64XN 32, 2 ; highbd_sad_skip_64x32_sse2
+HIGH_SAD64XN 16, 2 ; highbd_sad_skip_64x16_sse2
; unsigned int aom_highbd_sad32x{16,32,64}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro HIGH_SAD32XN 1-2 0
HIGH_SAD_FN 32, %1, 5, %2
+%if %2 == 2 ; skip rows, so divide number of rows by 2
+ mov n_rowsd, %1/2
+%else
mov n_rowsd, %1
+%endif
pxor m0, m0
pxor m6, m6
@@ -217,6 +247,9 @@
punpckldq m0, m6
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
@@ -225,17 +258,25 @@
HIGH_SAD32XN 64 ; highbd_sad32x64_sse2
HIGH_SAD32XN 32 ; highbd_sad32x32_sse2
HIGH_SAD32XN 16 ; highbd_sad32x16_sse2
+HIGH_SAD32XN 8 ; highbd_sad_32x8_sse2
HIGH_SAD32XN 64, 1 ; highbd_sad32x64_avg_sse2
HIGH_SAD32XN 32, 1 ; highbd_sad32x32_avg_sse2
HIGH_SAD32XN 16, 1 ; highbd_sad32x16_avg_sse2
-HIGH_SAD32XN 8 ; highbd_sad_32x8_sse2
-HIGH_SAD32XN 8, 1 ; highbd_sad_32x8_avg_sse2
+HIGH_SAD32XN 8, 1 ; highbd_sad_32x8_avg_sse2
+HIGH_SAD32XN 64, 2 ; highbd_sad_skip_32x64_sse2
+HIGH_SAD32XN 32, 2 ; highbd_sad_skip_32x32_sse2
+HIGH_SAD32XN 16, 2 ; highbd_sad_skip_32x16_sse2
+HIGH_SAD32XN 8, 2 ; highbd_sad_skip_32x8_sse2
; unsigned int aom_highbd_sad16x{8,16,32}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro HIGH_SAD16XN 1-2 0
HIGH_SAD_FN 16, %1, 5, %2
+%if %2 == 2 ; skip rows, so divide number of rows by 2
+ mov n_rowsd, %1/4
+%else
mov n_rowsd, %1/2
+%endif
pxor m0, m0
pxor m6, m6
@@ -287,27 +328,40 @@
punpckldq m0, m6
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
+HIGH_SAD16XN 64 ; highbd_sad_16x64_sse2
HIGH_SAD16XN 32 ; highbd_sad16x32_sse2
HIGH_SAD16XN 16 ; highbd_sad16x16_sse2
HIGH_SAD16XN 8 ; highbd_sad16x8_sse2
+HIGH_SAD16XN 4 ; highbd_sad_16x4_sse2
+HIGH_SAD16XN 64, 1 ; highbd_sad_16x64_avg_sse2
HIGH_SAD16XN 32, 1 ; highbd_sad16x32_avg_sse2
HIGH_SAD16XN 16, 1 ; highbd_sad16x16_avg_sse2
HIGH_SAD16XN 8, 1 ; highbd_sad16x8_avg_sse2
-HIGH_SAD16XN 4 ; highbd_sad_16x4_sse2
-HIGH_SAD16XN 4, 1 ; highbd_sad_16x4_avg_sse2
-HIGH_SAD16XN 64 ; highbd_sad_16x64_sse2
-HIGH_SAD16XN 64, 1 ; highbd_sad_16x64_avg_sse2
+HIGH_SAD16XN 4, 1 ; highbd_sad_16x4_avg_sse2
+HIGH_SAD16XN 64, 2 ; highbd_sad_skip_16x64_sse2
+HIGH_SAD16XN 32, 2 ; highbd_sad_skip_16x32_sse2
+HIGH_SAD16XN 16, 2 ; highbd_sad_skip_16x16_sse2
+HIGH_SAD16XN 8, 2 ; highbd_sad_skip_16x8_sse2
+; Current code fails there are only 2 rows
+; HIGH_SAD16XN 4, 2 ; highbd_sad_skip_16x4_sse2
; unsigned int aom_highbd_sad8x{4,8,16}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro HIGH_SAD8XN 1-2 0
HIGH_SAD_FN 8, %1, 7, %2
+%if %2 == 2 ; skip rows, so divide number of rows by 2
+ mov n_rowsd, %1/8
+%else
mov n_rowsd, %1/4
+%endif
pxor m0, m0
pxor m6, m6
@@ -359,25 +413,37 @@
punpckldq m0, m6
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
+HIGH_SAD8XN 32 ; highbd_sad_8x32_sse2
HIGH_SAD8XN 16 ; highbd_sad8x16_sse2
HIGH_SAD8XN 8 ; highbd_sad8x8_sse2
HIGH_SAD8XN 4 ; highbd_sad8x4_sse2
+HIGH_SAD8XN 32, 1 ; highbd_sad_8x32_avg_sse2
HIGH_SAD8XN 16, 1 ; highbd_sad8x16_avg_sse2
HIGH_SAD8XN 8, 1 ; highbd_sad8x8_avg_sse2
HIGH_SAD8XN 4, 1 ; highbd_sad8x4_avg_sse2
-HIGH_SAD8XN 32 ; highbd_sad_8x32_sse2
-HIGH_SAD8XN 32, 1 ; highbd_sad_8x32_avg_sse2
+HIGH_SAD8XN 32, 2 ; highbd_sad_skip_8x32_sse2
+HIGH_SAD8XN 16, 2 ; highbd_sad_skip_8x16_sse2
+HIGH_SAD8XN 8, 2 ; highbd_sad_skip_8x8_sse2
+; Current code fails there are only 2 rows
+; HIGH_SAD8XN 4, 2 ; highbd_sad8x4_avg_sse2
; unsigned int aom_highbd_sad4x{4,8,16}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro HIGH_SAD4XN 1-2 0
HIGH_SAD_FN 4, %1, 7, %2
+%if %2 == 2 ; skip rows, so divide number of rows by 2
+ mov n_rowsd, %1/8
+%else
mov n_rowsd, %1/4
+%endif
pxor m0, m0
pxor m6, m6
@@ -429,6 +495,9 @@
punpckldq m0, m6
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
@@ -440,3 +509,7 @@
HIGH_SAD4XN 16, 1 ; highbd_sad4x16_avg_sse2
HIGH_SAD4XN 8, 1 ; highbd_sad4x8_avg_sse2
HIGH_SAD4XN 4, 1 ; highbd_sad4x4_avg_sse2
+HIGH_SAD4XN 16, 2 ; highbd_sad_skip_4x16_sse2
+HIGH_SAD4XN 8, 2 ; highbd_sad_skip_4x8_sse2
+; Current code fails there are only 2 rows
+; HIGH_SAD4XN 4, 2 ; highbd_sad_skip_4x4_sse2
diff --git a/aom_dsp/x86/sad4d_avx2.c b/aom_dsp/x86/sad4d_avx2.c
index 0771252..0e0b904 100644
--- a/aom_dsp/x86/sad4d_avx2.c
+++ b/aom_dsp/x86/sad4d_avx2.c
@@ -104,3 +104,28 @@
sadMxN_avx2(128, 64);
sadMxN_avx2(128, 128);
+
+#define sad_skip_MxN_avx2(m, n) \
+ void aom_sad_skip_##m##x##n##x4d_avx2(const uint8_t *src, int src_stride, \
+ const uint8_t *const ref[4], \
+ int ref_stride, uint32_t res[4]) { \
+ aom_sadMxNx4d_avx2(m, ((n) >> 1), src, 2 * src_stride, ref, \
+ 2 * ref_stride, res); \
+ res[0] <<= 1; \
+ res[1] <<= 1; \
+ res[2] <<= 1; \
+ res[3] <<= 1; \
+ }
+
+sad_skip_MxN_avx2(32, 8);
+sad_skip_MxN_avx2(32, 16);
+sad_skip_MxN_avx2(32, 32);
+sad_skip_MxN_avx2(32, 64);
+
+sad_skip_MxN_avx2(64, 16);
+sad_skip_MxN_avx2(64, 32);
+sad_skip_MxN_avx2(64, 64);
+sad_skip_MxN_avx2(64, 128);
+
+sad_skip_MxN_avx2(128, 64);
+sad_skip_MxN_avx2(128, 128);
diff --git a/aom_dsp/x86/sad4d_sse2.asm b/aom_dsp/x86/sad4d_sse2.asm
index a904374..b5a17c5 100644
--- a/aom_dsp/x86/sad4d_sse2.asm
+++ b/aom_dsp/x86/sad4d_sse2.asm
@@ -312,9 +312,21 @@
; void aom_sadNxNx4d_sse2(uint8_t *src, int src_stride,
; uint8_t *ref[4], int ref_stride,
; uint32_t res[4]);
-; where NxN = 64x64, 32x32, 16x16, 16x8, 8x16, 8x8, 8x4, 4x8 and 4x4
-%macro SADNXN4D 2-3 0
-%if %3 == 0
+; Macro Arguments:
+; 1: Width
+; 2: Height
+; 3: If 0, then normal sad, else avg
+; 4: If 0, then normal sad, else skip rows
+%macro SADNXN4D 2-4 0,0
+%if %4 == 1 ; skip rows
+%if UNIX64
+cglobal sad_skip_%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \
+ res, ref2, ref3, ref4
+%else
+cglobal sad_skip_%1x%2x4d, 4, 7, 8, src, src_stride, ref1, ref_stride, \
+ ref2, ref3, ref4
+%endif
+%elif %3 == 0 ; normal sad
%if UNIX64
cglobal sad%1x%2x4d, 5, 8, 8, src, src_stride, ref1, ref_stride, \
res, ref2, ref3, ref4
@@ -323,7 +335,6 @@
ref2, ref3, ref4
%endif
%else ; avg
-
%if UNIX64
cglobal sad%1x%2x4d_avg, 6, 10, 8, src, src_stride, ref1, ref_stride, \
second_pred, res, ref2, ref3, ref4
@@ -336,6 +347,10 @@
%endif
%define mflag ((1 - UNIX64) & %3)
+%if %4 == 1
+ lea src_strided, [2*src_strided]
+ lea ref_strided, [2*ref_strided]
+%endif
movsxdifnidn src_strideq, src_strided
movsxdifnidn ref_strideq, ref_strided
@@ -345,9 +360,15 @@
mov ref1q, [ref1q+gprsize*0]
PROCESS_%1x2x4 1, 0, 0, 0, ref_strideq, %3, 0, 1, 2
-%rep (%2-4)/2
+%if %4 == 1 ; downsample number of rows by 2
+%define num_rep (%2-8)/4
+%else
+%define num_rep (%2-4)/2
+%endif
+%rep num_rep
PROCESS_%1x2x4 0, 0, 0, 0, ref_strideq, %3, 0, 1, 2
%endrep
+%undef num_rep
PROCESS_%1x2x4 0, 0, 0, 0, ref_strideq, %3, 0, 1, 2
%if %3 == 0
@@ -368,12 +389,19 @@
punpcklqdq m4, m6
punpckhqdq m5, m7
paddd m4, m5
+%if %4 == 1
+ pslld m4, 1
+%endif
movifnidn resultq, resultmp
movu [resultq], m4
RET
%else
pshufd m6, m6, 0x08
pshufd m7, m7, 0x08
+%if %4 == 1
+ pslld m6, 1
+ pslld m7, 1
+%endif
movifnidn resultq, resultmp
movq [resultq+0], m6
movq [resultq+8], m7
@@ -383,46 +411,70 @@
INIT_XMM sse2
SADNXN4D 128, 128
-SADNXN4D 128, 64
-SADNXN4D 64, 128
-SADNXN4D 64, 64
-SADNXN4D 64, 32
-SADNXN4D 32, 64
-SADNXN4D 32, 32
-SADNXN4D 32, 16
-SADNXN4D 16, 32
-SADNXN4D 16, 16
-SADNXN4D 16, 8
-SADNXN4D 8, 16
-SADNXN4D 8, 8
-SADNXN4D 8, 4
-SADNXN4D 4, 8
-SADNXN4D 4, 4
-SADNXN4D 4, 16
-SADNXN4D 16, 4
-SADNXN4D 8, 32
-SADNXN4D 32, 8
-SADNXN4D 16, 64
-SADNXN4D 64, 16
+SADNXN4D 128, 64
+SADNXN4D 64, 128
+SADNXN4D 64, 64
+SADNXN4D 64, 32
+SADNXN4D 32, 64
+SADNXN4D 32, 32
+SADNXN4D 32, 16
+SADNXN4D 16, 32
+SADNXN4D 16, 16
+SADNXN4D 16, 8
+SADNXN4D 8, 16
+SADNXN4D 8, 8
+SADNXN4D 8, 4
+SADNXN4D 4, 8
+SADNXN4D 4, 4
+SADNXN4D 4, 16
+SADNXN4D 16, 4
+SADNXN4D 8, 32
+SADNXN4D 32, 8
+SADNXN4D 16, 64
+SADNXN4D 64, 16
SADNXN4D 128, 128, 1
-SADNXN4D 128, 64, 1
-SADNXN4D 64, 128, 1
-SADNXN4D 64, 64, 1
-SADNXN4D 64, 32, 1
-SADNXN4D 32, 64, 1
-SADNXN4D 32, 32, 1
-SADNXN4D 32, 16, 1
-SADNXN4D 16, 32, 1
-SADNXN4D 16, 16, 1
-SADNXN4D 16, 8, 1
-SADNXN4D 8, 16, 1
-SADNXN4D 8, 8, 1
-SADNXN4D 8, 4, 1
-SADNXN4D 4, 8, 1
-SADNXN4D 4, 4, 1
-SADNXN4D 4, 16, 1
-SADNXN4D 16, 4, 1
-SADNXN4D 8, 32, 1
-SADNXN4D 32, 8, 1
-SADNXN4D 16, 64, 1
-SADNXN4D 64, 16, 1
+SADNXN4D 128, 64, 1
+SADNXN4D 64, 128, 1
+SADNXN4D 64, 64, 1
+SADNXN4D 64, 32, 1
+SADNXN4D 32, 64, 1
+SADNXN4D 32, 32, 1
+SADNXN4D 32, 16, 1
+SADNXN4D 16, 32, 1
+SADNXN4D 16, 16, 1
+SADNXN4D 16, 8, 1
+SADNXN4D 8, 16, 1
+SADNXN4D 8, 8, 1
+SADNXN4D 8, 4, 1
+SADNXN4D 4, 8, 1
+SADNXN4D 4, 4, 1
+SADNXN4D 4, 16, 1
+SADNXN4D 16, 4, 1
+SADNXN4D 8, 32, 1
+SADNXN4D 32, 8, 1
+SADNXN4D 16, 64, 1
+SADNXN4D 64, 16, 1
+SADNXN4D 128, 128, 0, 1
+SADNXN4D 128, 64, 0, 1
+SADNXN4D 64, 128, 0, 1
+SADNXN4D 64, 64, 0, 1
+SADNXN4D 64, 32, 0, 1
+SADNXN4D 32, 64, 0, 1
+SADNXN4D 32, 32, 0, 1
+SADNXN4D 32, 16, 0, 1
+SADNXN4D 16, 32, 0, 1
+SADNXN4D 16, 16, 0, 1
+SADNXN4D 16, 8, 0, 1
+SADNXN4D 8, 16, 0, 1
+SADNXN4D 8, 8, 0, 1
+SADNXN4D 4, 8, 0, 1
+SADNXN4D 4, 16, 0, 1
+SADNXN4D 8, 32, 0, 1
+SADNXN4D 32, 8, 0, 1
+SADNXN4D 16, 64, 0, 1
+SADNXN4D 64, 16, 0, 1
+
+; Different assembly is needed when the height gets subsampled to 2
+; SADNXN4D 16, 4, 0, 1
+; SADNXN4D 8, 4, 0, 1
+; SADNXN4D 4, 4, 0, 1
diff --git a/aom_dsp/x86/sad_avx2.c b/aom_dsp/x86/sad_avx2.c
index a50dba6..9ab9812 100644
--- a/aom_dsp/x86/sad_avx2.c
+++ b/aom_dsp/x86/sad_avx2.c
@@ -14,76 +14,106 @@
#include "aom_ports/mem.h"
+static INLINE unsigned int sad64xh_avx2(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride,
+ int h) {
+ int i, res;
+ __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg;
+ __m256i sum_sad = _mm256_setzero_si256();
+ __m256i sum_sad_h;
+ __m128i sum_sad128;
+ for (i = 0; i < h; i++) {
+ ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr);
+ ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32));
+ sad1_reg =
+ _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr));
+ sad2_reg = _mm256_sad_epu8(
+ ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32)));
+ sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg));
+ ref_ptr += ref_stride;
+ src_ptr += src_stride;
+ }
+ sum_sad_h = _mm256_srli_si256(sum_sad, 8);
+ sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h);
+ sum_sad128 = _mm256_extracti128_si256(sum_sad, 1);
+ sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128);
+ res = _mm_cvtsi128_si32(sum_sad128);
+ _mm256_zeroupper();
+ return res;
+}
+
+static INLINE unsigned int sad32xh_avx2(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride,
+ int h) {
+ int i, res;
+ __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg;
+ __m256i sum_sad = _mm256_setzero_si256();
+ __m256i sum_sad_h;
+ __m128i sum_sad128;
+ int ref2_stride = ref_stride << 1;
+ int src2_stride = src_stride << 1;
+ int max = h >> 1;
+ for (i = 0; i < max; i++) {
+ ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr);
+ ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride));
+ sad1_reg =
+ _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr));
+ sad2_reg = _mm256_sad_epu8(
+ ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride)));
+ sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg));
+ ref_ptr += ref2_stride;
+ src_ptr += src2_stride;
+ }
+ sum_sad_h = _mm256_srli_si256(sum_sad, 8);
+ sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h);
+ sum_sad128 = _mm256_extracti128_si256(sum_sad, 1);
+ sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128);
+ res = _mm_cvtsi128_si32(sum_sad128);
+ _mm256_zeroupper();
+ return res;
+}
+
#define FSAD64_H(h) \
unsigned int aom_sad64x##h##_avx2(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride) { \
- int i, res; \
- __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \
- __m256i sum_sad = _mm256_setzero_si256(); \
- __m256i sum_sad_h; \
- __m128i sum_sad128; \
- for (i = 0; i < h; i++) { \
- ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \
- ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); \
- sad1_reg = _mm256_sad_epu8( \
- ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \
- sad2_reg = _mm256_sad_epu8( \
- ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); \
- sum_sad = \
- _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \
- ref_ptr += ref_stride; \
- src_ptr += src_stride; \
- } \
- sum_sad_h = _mm256_srli_si256(sum_sad, 8); \
- sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \
- sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \
- sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \
- res = _mm_cvtsi128_si32(sum_sad128); \
- _mm256_zeroupper(); \
- return res; \
+ return sad64xh_avx2(src_ptr, src_stride, ref_ptr, ref_stride, h); \
+ }
+
+#define FSADS64_H(h) \
+ unsigned int aom_sad_skip_64x##h##_avx2( \
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
+ int ref_stride) { \
+ return 2 * sad64xh_avx2(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \
+ h / 2); \
}
#define FSAD32_H(h) \
unsigned int aom_sad32x##h##_avx2(const uint8_t *src_ptr, int src_stride, \
const uint8_t *ref_ptr, int ref_stride) { \
- int i, res; \
- __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \
- __m256i sum_sad = _mm256_setzero_si256(); \
- __m256i sum_sad_h; \
- __m128i sum_sad128; \
- int ref2_stride = ref_stride << 1; \
- int src2_stride = src_stride << 1; \
- int max = h >> 1; \
- for (i = 0; i < max; i++) { \
- ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \
- ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); \
- sad1_reg = _mm256_sad_epu8( \
- ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \
- sad2_reg = _mm256_sad_epu8( \
- ref2_reg, \
- _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); \
- sum_sad = \
- _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \
- ref_ptr += ref2_stride; \
- src_ptr += src2_stride; \
- } \
- sum_sad_h = _mm256_srli_si256(sum_sad, 8); \
- sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \
- sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \
- sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \
- res = _mm_cvtsi128_si32(sum_sad128); \
- _mm256_zeroupper(); \
- return res; \
+ return sad32xh_avx2(src_ptr, src_stride, ref_ptr, ref_stride, h); \
}
-#define FSAD64 \
- FSAD64_H(64); \
- FSAD64_H(32);
+#define FSADS32_H(h) \
+ unsigned int aom_sad_skip_32x##h##_avx2( \
+ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \
+ int ref_stride) { \
+ return 2 * sad32xh_avx2(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \
+ h / 2); \
+ }
-#define FSAD32 \
- FSAD32_H(64); \
- FSAD32_H(32); \
- FSAD32_H(16);
+#define FSAD64 \
+ FSAD64_H(64); \
+ FSAD64_H(32); \
+ FSADS64_H(64); \
+ FSADS64_H(32);
+
+#define FSAD32 \
+ FSAD32_H(64); \
+ FSAD32_H(32); \
+ FSAD32_H(16); \
+ FSADS32_H(64); \
+ FSADS32_H(32); \
+ FSADS32_H(16);
/* clang-format off */
FSAD64
diff --git a/aom_dsp/x86/sad_highbd_avx2.c b/aom_dsp/x86/sad_highbd_avx2.c
index 2cff2e6..7d0626a 100644
--- a/aom_dsp/x86/sad_highbd_avx2.c
+++ b/aom_dsp/x86/sad_highbd_avx2.c
@@ -259,6 +259,14 @@
return aom_highbd_sad##m##xN_avx2(n, src, src_stride, ref, ref_stride); \
}
+#define highbd_sad_skip_MxN_avx2(m, n) \
+ unsigned int aom_highbd_sad_skip_##m##x##n##_avx2( \
+ const uint8_t *src, int src_stride, const uint8_t *ref, \
+ int ref_stride) { \
+ return 2 * aom_highbd_sad##m##xN_avx2((n / 2), src, 2 * src_stride, ref, \
+ 2 * ref_stride); \
+ }
+
highbd_sadMxN_avx2(16, 4);
highbd_sadMxN_avx2(16, 8);
highbd_sadMxN_avx2(16, 16);
@@ -278,6 +286,24 @@
highbd_sadMxN_avx2(128, 64);
highbd_sadMxN_avx2(128, 128);
+highbd_sad_skip_MxN_avx2(16, 8);
+highbd_sad_skip_MxN_avx2(16, 16);
+highbd_sad_skip_MxN_avx2(16, 32);
+highbd_sad_skip_MxN_avx2(16, 64);
+
+highbd_sad_skip_MxN_avx2(32, 8);
+highbd_sad_skip_MxN_avx2(32, 16);
+highbd_sad_skip_MxN_avx2(32, 32);
+highbd_sad_skip_MxN_avx2(32, 64);
+
+highbd_sad_skip_MxN_avx2(64, 16);
+highbd_sad_skip_MxN_avx2(64, 32);
+highbd_sad_skip_MxN_avx2(64, 64);
+highbd_sad_skip_MxN_avx2(64, 128);
+
+highbd_sad_skip_MxN_avx2(128, 64);
+highbd_sad_skip_MxN_avx2(128, 128);
+
unsigned int aom_highbd_sad16x4_avg_avx2(const uint8_t *src, int src_stride,
const uint8_t *ref, int ref_stride,
const uint8_t *second_pred) {
@@ -678,6 +704,17 @@
aom_highbd_sad##m##xNx4d_avx2(n, src, src_stride, ref_array, ref_stride, \
sad_array); \
}
+#define highbd_sad_skip_MxNx4d_avx2(m, n) \
+ void aom_highbd_sad_skip_##m##x##n##x4d_avx2( \
+ const uint8_t *src, int src_stride, const uint8_t *const ref_array[], \
+ int ref_stride, uint32_t *sad_array) { \
+ aom_highbd_sad##m##xNx4d_avx2((n / 2), src, 2 * src_stride, ref_array, \
+ 2 * ref_stride, sad_array); \
+ sad_array[0] <<= 1; \
+ sad_array[1] <<= 1; \
+ sad_array[2] <<= 1; \
+ sad_array[3] <<= 1; \
+ }
highbd_sadMxNx4d_avx2(16, 4);
highbd_sadMxNx4d_avx2(16, 8);
@@ -697,3 +734,21 @@
highbd_sadMxNx4d_avx2(128, 64);
highbd_sadMxNx4d_avx2(128, 128);
+
+highbd_sad_skip_MxNx4d_avx2(16, 8);
+highbd_sad_skip_MxNx4d_avx2(16, 16);
+highbd_sad_skip_MxNx4d_avx2(16, 32);
+highbd_sad_skip_MxNx4d_avx2(16, 64);
+
+highbd_sad_skip_MxNx4d_avx2(32, 8);
+highbd_sad_skip_MxNx4d_avx2(32, 16);
+highbd_sad_skip_MxNx4d_avx2(32, 32);
+highbd_sad_skip_MxNx4d_avx2(32, 64);
+
+highbd_sad_skip_MxNx4d_avx2(64, 16);
+highbd_sad_skip_MxNx4d_avx2(64, 32);
+highbd_sad_skip_MxNx4d_avx2(64, 64);
+highbd_sad_skip_MxNx4d_avx2(64, 128);
+
+highbd_sad_skip_MxNx4d_avx2(128, 64);
+highbd_sad_skip_MxNx4d_avx2(128, 128);
diff --git a/aom_dsp/x86/sad_impl_avx2.c b/aom_dsp/x86/sad_impl_avx2.c
index f77a585..2afae4b 100644
--- a/aom_dsp/x86/sad_impl_avx2.c
+++ b/aom_dsp/x86/sad_impl_avx2.c
@@ -84,6 +84,30 @@
return sum;
}
+unsigned int aom_sad_skip_128x64_avx2(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride) {
+ const uint32_t half_width = 64;
+ uint32_t sum = sad64x32(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2);
+ src_ptr += half_width;
+ ref_ptr += half_width;
+ sum += sad64x32(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2);
+ return 2 * sum;
+}
+
+unsigned int aom_sad_skip_64x128_avx2(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride) {
+ const uint32_t sum =
+ sad64x64(src_ptr, 2 * src_stride, ref_ptr, 2 * ref_stride);
+ return 2 * sum;
+}
+
+unsigned int aom_sad_skip_128x128_avx2(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride) {
+ const uint32_t sum =
+ aom_sad128x64_avx2(src_ptr, 2 * src_stride, ref_ptr, 2 * ref_stride);
+ return 2 * sum;
+}
+
static unsigned int sad_w64_avg_avx2(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
const int h, const uint8_t *second_pred,
diff --git a/aom_dsp/x86/sad_sse2.asm b/aom_dsp/x86/sad_sse2.asm
index 3251b76..de9845a 100644
--- a/aom_dsp/x86/sad_sse2.asm
+++ b/aom_dsp/x86/sad_sse2.asm
@@ -15,15 +15,29 @@
SECTION .text
+; Macro Arguments
+; Arg 1: Width
+; Arg 2: Height
+; Arg 3: Number of general purpose registers: 5 for 32-bit build, 6 for 64-bit
+; Arg 4: Type of function: if 0, normal sad; if 1, avg; if 2, skip rows
%macro SAD_FN 4
-%if %4 == 0
+%if %4 == 0 ; normal sad
%if %3 == 5
cglobal sad%1x%2, 4, %3, 5, src, src_stride, ref, ref_stride, n_rows
%else ; %3 == 7
cglobal sad%1x%2, 4, %3, 6, src, src_stride, ref, ref_stride, \
src_stride3, ref_stride3, n_rows
%endif ; %3 == 5/7
-%else ; avg
+
+%elif %4 == 2 ; skip
+%if %3 == 5
+cglobal sad_skip_%1x%2, 4, %3, 5, src, src_stride, ref, ref_stride, n_rows
+%else ; %3 == 7
+cglobal sad_skip_%1x%2, 4, %3, 6, src, src_stride, ref, ref_stride, \
+ src_stride3, ref_stride3, n_rows
+%endif ; %3 == 5/7
+
+%else
%if %3 == 5
cglobal sad%1x%2_avg, 5, 1 + %3, 5, src, src_stride, ref, ref_stride, \
second_pred, n_rows
@@ -38,7 +52,11 @@
%define n_rowsd dword r0m
%endif ; x86-32/64
%endif ; %3 == 5/7
-%endif ; avg/sad
+%endif ; sad/avg/skip
+%if %4 == 2; skip rows so double the stride
+lea src_strided, [src_strided*2]
+lea ref_strided, [ref_strided*2]
+%endif ; %4 skip
movsxdifnidn src_strideq, src_strided
movsxdifnidn ref_strideq, ref_strided
%if %3 == 7
@@ -51,7 +69,11 @@
; uint8_t *ref, int ref_stride);
%macro SAD128XN 1-2 0
SAD_FN 128, %1, 5, %2
+%if %2 == 2
+ mov n_rowsd, %1/2
+%else
mov n_rowsd, %1
+%endif
pxor m0, m0
.loop:
@@ -104,6 +126,9 @@
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so now we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
@@ -111,15 +136,21 @@
INIT_XMM sse2
SAD128XN 128 ; sad128x128_sse2
SAD128XN 128, 1 ; sad128x128_avg_sse2
+SAD128XN 128, 2 ; sad128x128_skip_sse2
SAD128XN 64 ; sad128x64_sse2
SAD128XN 64, 1 ; sad128x64_avg_sse2
+SAD128XN 64, 2 ; sad128x64_skip_sse2
; unsigned int aom_sad64x64_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro SAD64XN 1-2 0
SAD_FN 64, %1, 5, %2
+%if %2 == 2
+ mov n_rowsd, %1/2
+%else
mov n_rowsd, %1
+%endif
pxor m0, m0
.loop:
movu m1, [refq]
@@ -148,25 +179,36 @@
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so now we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
SAD64XN 128 ; sad64x128_sse2
+SAD64XN 64 ; sad64x64_sse2
+SAD64XN 32 ; sad64x32_sse2
+SAD64XN 16 ; sad64x16_sse2
SAD64XN 128, 1 ; sad64x128_avg_sse2
-SAD64XN 64 ; sad64x64_sse2
-SAD64XN 32 ; sad64x32_sse2
-SAD64XN 64, 1 ; sad64x64_avg_sse2
-SAD64XN 32, 1 ; sad64x32_avg_sse2
-SAD64XN 16 ; sad64x16_sse2
-SAD64XN 16, 1 ; sad64x16_avg_sse2
+SAD64XN 64, 1 ; sad64x64_avg_sse2
+SAD64XN 32, 1 ; sad64x32_avg_sse2
+SAD64XN 16, 1 ; sad64x16_avg_sse2
+SAD64XN 128, 2 ; sad64x128_skip_sse2
+SAD64XN 64, 2 ; sad64x64_skip_sse2
+SAD64XN 32, 2 ; sad64x32_skip_sse2
+SAD64XN 16, 2 ; sad64x16_skip_sse2
; unsigned int aom_sad32x32_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro SAD32XN 1-2 0
SAD_FN 32, %1, 5, %2
+%if %2 == 2
+ mov n_rowsd, %1/4
+%else
mov n_rowsd, %1/2
+%endif
pxor m0, m0
.loop:
movu m1, [refq]
@@ -195,25 +237,36 @@
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so now we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
-SAD32XN 64 ; sad32x64_sse2
-SAD32XN 32 ; sad32x32_sse2
-SAD32XN 16 ; sad32x16_sse2
+SAD32XN 64 ; sad32x64_sse2
+SAD32XN 32 ; sad32x32_sse2
+SAD32XN 16 ; sad32x16_sse2
+SAD32XN 8 ; sad_32x8_sse2
SAD32XN 64, 1 ; sad32x64_avg_sse2
SAD32XN 32, 1 ; sad32x32_avg_sse2
SAD32XN 16, 1 ; sad32x16_avg_sse2
-SAD32XN 8 ; sad_32x8_sse2
-SAD32XN 8, 1 ; sad_32x8_avg_sse2
+SAD32XN 8, 1 ; sad_32x8_avg_sse2
+SAD32XN 64, 2 ; sad32x64_skip_sse2
+SAD32XN 32, 2 ; sad32x32_skip_sse2
+SAD32XN 16, 2 ; sad32x16_skip_sse2
+SAD32XN 8, 2 ; sad_32x8_skip_sse2
; unsigned int aom_sad16x{8,16}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro SAD16XN 1-2 0
SAD_FN 16, %1, 7, %2
+%if %2 == 2
+ mov n_rowsd, %1/8
+%else
mov n_rowsd, %1/4
+%endif
pxor m0, m0
.loop:
@@ -243,27 +296,38 @@
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so now we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
-SAD16XN 32 ; sad16x32_sse2
-SAD16XN 16 ; sad16x16_sse2
-SAD16XN 8 ; sad16x8_sse2
+SAD16XN 64 ; sad_16x64_sse2
+SAD16XN 32 ; sad16x32_sse2
+SAD16XN 16 ; sad16x16_sse2
+SAD16XN 8 ; sad16x8_sse2
+SAD16XN 4 ; sad_16x4_sse2
+SAD16XN 64, 1 ; sad_16x64_avg_sse2
SAD16XN 32, 1 ; sad16x32_avg_sse2
SAD16XN 16, 1 ; sad16x16_avg_sse2
SAD16XN 8, 1 ; sad16x8_avg_sse2
-SAD16XN 4 ; sad_16x4_sse2
-SAD16XN 4, 1 ; sad_16x4_avg_sse2
-SAD16XN 64 ; sad_16x64_sse2
-SAD16XN 64, 1 ; sad_16x64_avg_sse2
+SAD16XN 4, 1 ; sad_16x4_avg_sse2
+SAD16XN 64, 2 ; sad_16x64_skip_sse2
+SAD16XN 32, 2 ; sad16x32_skip_sse2
+SAD16XN 16, 2 ; sad16x16_skip_sse2
+SAD16XN 8, 2 ; sad16x8_skip_sse2
; unsigned int aom_sad8x{8,16}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro SAD8XN 1-2 0
SAD_FN 8, %1, 7, %2
+%if %2 == 2
+ mov n_rowsd, %1/8
+%else
mov n_rowsd, %1/4
+%endif
pxor m0, m0
.loop:
@@ -291,25 +355,35 @@
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so now we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
-SAD8XN 16 ; sad8x16_sse2
-SAD8XN 8 ; sad8x8_sse2
-SAD8XN 4 ; sad8x4_sse2
+SAD8XN 32 ; sad_8x32_sse2
+SAD8XN 16 ; sad8x16_sse2
+SAD8XN 8 ; sad8x8_sse2
+SAD8XN 4 ; sad8x4_sse2
+SAD8XN 32, 1 ; sad_8x32_avg_sse2
SAD8XN 16, 1 ; sad8x16_avg_sse2
SAD8XN 8, 1 ; sad8x8_avg_sse2
SAD8XN 4, 1 ; sad8x4_avg_sse2
-SAD8XN 32 ; sad_8x32_sse2
-SAD8XN 32, 1 ; sad_8x32_avg_sse2
+SAD8XN 32, 2 ; sad_8x32_skip_sse2
+SAD8XN 16, 2 ; sad8x16_skip_sse2
+SAD8XN 8, 2 ; sad8x8_skip_sse2
; unsigned int aom_sad4x{4, 8}_sse2(uint8_t *src, int src_stride,
; uint8_t *ref, int ref_stride);
%macro SAD4XN 1-2 0
SAD_FN 4, %1, 7, %2
+%if %2 == 2
+ mov n_rowsd, %1/8
+%else
mov n_rowsd, %1/4
+%endif
pxor m0, m0
.loop:
@@ -340,14 +414,19 @@
movhlps m1, m0
paddd m0, m1
+%if %2 == 2 ; we skipped rows, so now we need to double the sad
+ pslld m0, 1
+%endif
movd eax, m0
RET
%endmacro
INIT_XMM sse2
+SAD4XN 16 ; sad_4x16_sse2
SAD4XN 8 ; sad4x8_sse
SAD4XN 4 ; sad4x4_sse
+SAD4XN 16, 1 ; sad_4x16_avg_sse2
SAD4XN 8, 1 ; sad4x8_avg_sse
SAD4XN 4, 1 ; sad4x4_avg_sse
-SAD4XN 16 ; sad_4x16_sse2
-SAD4XN 16, 1 ; sad_4x16_avg_sse2
+SAD4XN 16, 2 ; sad_4x16_skip_sse2
+SAD4XN 8, 2 ; sad4x8_skip_sse
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index b829e55..a4e6497 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -15,6 +15,7 @@
#include <stdio.h>
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#if CONFIG_DENOISE
#include "aom_dsp/grain_table.h"
@@ -1243,6 +1244,35 @@
MBFP(BLOCK_64X16, aom_masked_sad64x16, aom_masked_sub_pixel_variance64x16)
+#define SDSFP(BT, SDSF, SDSX4DF) \
+ cpi->fn_ptr[BT].sdsf = SDSF; \
+ cpi->fn_ptr[BT].sdsx4df = SDSX4DF;
+
+ SDSFP(BLOCK_128X128, aom_sad_skip_128x128, aom_sad_skip_128x128x4d);
+ SDSFP(BLOCK_128X64, aom_sad_skip_128x64, aom_sad_skip_128x64x4d);
+ SDSFP(BLOCK_64X128, aom_sad_skip_64x128, aom_sad_skip_64x128x4d);
+ SDSFP(BLOCK_64X64, aom_sad_skip_64x64, aom_sad_skip_64x64x4d);
+ SDSFP(BLOCK_64X32, aom_sad_skip_64x32, aom_sad_skip_64x32x4d);
+ SDSFP(BLOCK_64X16, aom_sad_skip_64x16, aom_sad_skip_64x16x4d);
+ SDSFP(BLOCK_32X64, aom_sad_skip_32x64, aom_sad_skip_32x64x4d);
+ SDSFP(BLOCK_32X32, aom_sad_skip_32x32, aom_sad_skip_32x32x4d);
+ SDSFP(BLOCK_32X16, aom_sad_skip_32x16, aom_sad_skip_32x16x4d);
+ SDSFP(BLOCK_32X8, aom_sad_skip_32x8, aom_sad_skip_32x8x4d);
+
+ SDSFP(BLOCK_16X64, aom_sad_skip_16x64, aom_sad_skip_16x64x4d);
+ SDSFP(BLOCK_16X32, aom_sad_skip_16x32, aom_sad_skip_16x32x4d);
+ SDSFP(BLOCK_16X16, aom_sad_skip_16x16, aom_sad_skip_16x16x4d);
+ SDSFP(BLOCK_16X8, aom_sad_skip_16x8, aom_sad_skip_16x8x4d);
+ SDSFP(BLOCK_8X16, aom_sad_skip_8x16, aom_sad_skip_8x16x4d);
+ SDSFP(BLOCK_8X8, aom_sad_skip_8x8, aom_sad_skip_8x8x4d);
+ SDSFP(BLOCK_4X16, aom_sad_skip_4x16, aom_sad_skip_4x16x4d);
+ SDSFP(BLOCK_4X8, aom_sad_skip_4x8, aom_sad_skip_4x8x4d);
+ SDSFP(BLOCK_4X16, aom_sad_skip_4x16, aom_sad_skip_4x16x4d);
+ SDSFP(BLOCK_8X32, aom_sad_skip_8x32, aom_sad_skip_8x32x4d);
+ SDSFP(BLOCK_32X8, aom_sad_skip_32x8, aom_sad_skip_32x8x4d);
+ SDSFP(BLOCK_64X16, aom_sad_skip_64x16, aom_sad_skip_64x16x4d);
+#undef SDSFP
+
#if CONFIG_AV1_HIGHBITDEPTH
highbd_set_var_fns(cpi);
#endif
diff --git a/av1/encoder/encoder_utils.h b/av1/encoder/encoder_utils.h
index 4cb480a3..800cc8d 100644
--- a/av1/encoder/encoder_utils.h
+++ b/av1/encoder/encoder_utils.h
@@ -329,6 +329,15 @@
aom_highbd_masked_sad##WIDTH##x##HEIGHT##_bits##BD, \
aom_highbd_##BD##_masked_sub_pixel_variance##WIDTH##x##HEIGHT)
+#define HIGHBD_SDSFP(BT, SDSF, SDSX4DF) \
+ cpi->fn_ptr[BT].sdsf = SDSF; \
+ cpi->fn_ptr[BT].sdsx4df = SDSX4DF;
+
+#define HIGHBD_SDSFP_WRAPPER(WIDTH, HEIGHT) \
+ HIGHBD_SDSFP(BLOCK_##WIDTH##X##HEIGHT, \
+ aom_highbd_sad_skip_##WIDTH##x##HEIGHT, \
+ aom_highbd_sad_skip_##WIDTH##x##HEIGHT##x4d)
+
#define MAKE_MBFP_COMPOUND_SAD_WRAPPER(fnname) \
static unsigned int fnname##_bits8( \
const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, \
@@ -659,6 +668,29 @@
"cm->seq_params.bit_depth should be AOM_BITS_8, "
"AOM_BITS_10 or AOM_BITS_12");
}
+
+ HIGHBD_SDSFP_WRAPPER(128, 128);
+ HIGHBD_SDSFP_WRAPPER(128, 64);
+ HIGHBD_SDSFP_WRAPPER(64, 128);
+ HIGHBD_SDSFP_WRAPPER(64, 64);
+ HIGHBD_SDSFP_WRAPPER(64, 32);
+ HIGHBD_SDSFP_WRAPPER(64, 16);
+ HIGHBD_SDSFP_WRAPPER(32, 64);
+ HIGHBD_SDSFP_WRAPPER(32, 32);
+ HIGHBD_SDSFP_WRAPPER(32, 16);
+ HIGHBD_SDSFP_WRAPPER(32, 8);
+ HIGHBD_SDSFP_WRAPPER(16, 64);
+ HIGHBD_SDSFP_WRAPPER(16, 32);
+ HIGHBD_SDSFP_WRAPPER(16, 16);
+ HIGHBD_SDSFP_WRAPPER(16, 8);
+ HIGHBD_SDSFP_WRAPPER(8, 16);
+ HIGHBD_SDSFP_WRAPPER(8, 8);
+ HIGHBD_SDSFP_WRAPPER(4, 16);
+ HIGHBD_SDSFP_WRAPPER(4, 8);
+ HIGHBD_SDSFP_WRAPPER(4, 16);
+ HIGHBD_SDSFP_WRAPPER(8, 32);
+ HIGHBD_SDSFP_WRAPPER(32, 8);
+ HIGHBD_SDSFP_WRAPPER(64, 16);
}
}
#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/av1/encoder/mcomp.c b/av1/encoder/mcomp.c
index 4225b00..7c8457f 100644
--- a/av1/encoder/mcomp.c
+++ b/av1/encoder/mcomp.c
@@ -99,6 +99,16 @@
av1_set_mv_search_method(ms_params, search_sites, search_method);
+ const int use_downsampled_sad =
+ mv_sf->use_downsampled_sad && block_size_high[bsize] >= 16;
+ if (use_downsampled_sad) {
+ ms_params->sdf = ms_params->vfp->sdsf;
+ ms_params->sdx4df = ms_params->vfp->sdsx4df;
+ } else {
+ ms_params->sdf = ms_params->vfp->sdf;
+ ms_params->sdx4df = ms_params->vfp->sdx4df;
+ }
+
ms_params->mesh_patterns[0] = mv_sf->mesh_patterns;
ms_params->mesh_patterns[1] = mv_sf->intrabc_mesh_patterns;
ms_params->force_mesh_thresh = mv_sf->exhaustive_searches_thresh;
@@ -608,11 +618,10 @@
const struct buf_2d *const src,
const uint8_t *const ref_address,
const int ref_stride) {
- const aom_variance_fn_ptr_t *vfp = ms_params->vfp;
const uint8_t *src_buf = src->buf;
const int src_stride = src->stride;
- return vfp->sdf(src_buf, src_stride, ref_address, ref_stride);
+ return ms_params->sdf(src_buf, src_stride, ref_address, ref_stride);
}
static INLINE int get_mvpred_compound_var_cost(
@@ -668,7 +677,7 @@
} else if (second_pred) {
return vfp->sdaf(src_buf, src_stride, ref_address, ref_stride, second_pred);
} else {
- return vfp->sdf(src_buf, src_stride, ref_address, ref_stride);
+ return ms_params->sdf(src_buf, src_stride, ref_address, ref_stride);
}
}
@@ -808,7 +817,6 @@
const struct buf_2d *const ref = ms_params->ms_buffers.ref;
const search_site *site = ms_params->search_sites->site[search_step];
- const aom_variance_fn_ptr_t *vfp = ms_params->vfp;
unsigned char const *block_offset[4];
unsigned int sads[4];
const uint8_t *best_address;
@@ -819,8 +827,8 @@
for (int j = 0; j < 4; j++)
block_offset[j] = site[cand_start + j].offset + best_address;
- // 4-point sad calcuation.
- vfp->sdx4df(src_buf, src_stride, block_offset, ref->stride, sads);
+ // 4-point sad calculation.
+ ms_params->sdx4df(src_buf, src_stride, block_offset, ref->stride, sads);
for (int j = 0; j < 4; j++) {
const FULLPEL_MV this_mv = {
@@ -1219,7 +1227,6 @@
const int ref_stride = ref->stride;
const uint8_t *best_address;
- const aom_variance_fn_ptr_t *vfp = ms_params->vfp;
const uint8_t *mask = ms_params->ms_buffers.mask;
const uint8_t *second_pred = ms_params->ms_buffers.second_pred;
const MV_COST_PARAMS *mv_cost_params = &ms_params->mv_cost_params;
@@ -1268,7 +1275,7 @@
for (j = 0; j < 4; j++)
block_offset[j] = site[idx + j].offset + best_address;
- vfp->sdx4df(src_buf, src_stride, block_offset, ref_stride, sads);
+ ms_params->sdx4df(src_buf, src_stride, block_offset, ref_stride, sads);
for (j = 0; j < 4; j++) {
if (sads[j] < bestsad) {
const FULLPEL_MV this_mv = { best_mv->row + site[idx + j].mv.row,
@@ -1390,7 +1397,6 @@
const int range, const int step,
FULLPEL_MV *best_mv,
FULLPEL_MV *second_best_mv) {
- const aom_variance_fn_ptr_t *vfp = ms_params->vfp;
const MV_COST_PARAMS *mv_cost_params = &ms_params->mv_cost_params;
const struct buf_2d *const src = ms_params->ms_buffers.src;
const struct buf_2d *const ref = ms_params->ms_buffers.ref;
@@ -1430,7 +1436,8 @@
const FULLPEL_MV mv = { start_mv.row + r, start_mv.col + c + i };
addrs[i] = get_buf_from_fullmv(ref, &mv);
}
- vfp->sdx4df(src->buf, src->stride, addrs, ref_stride, sads);
+
+ ms_params->sdx4df(src->buf, src->stride, addrs, ref_stride, sads);
for (i = 0; i < 4; ++i) {
if (sads[i] < best_sad) {
@@ -1683,6 +1690,40 @@
}
}
+ if (ms_params->sdf != ms_params->vfp->sdf) {
+ // If we are skipping rows when we perform the motion search, we need to
+ // check the quality of skipping. If it's bad, then we run mesh search with
+ // skip row features off.
+ // TODO(chiyotsai@google.com): Handle the case where we have a vertical
+ // offset of 1 before we hit this statement to avoid having to redo
+ // motion search.
+ const struct buf_2d *src = ms_params->ms_buffers.src;
+ const struct buf_2d *ref = ms_params->ms_buffers.ref;
+ const int src_stride = src->stride;
+ const int ref_stride = ref->stride;
+
+ const uint8_t *src_address = src->buf;
+ const uint8_t *best_address = get_buf_from_fullmv(ref, best_mv);
+ const int sad =
+ ms_params->vfp->sdf(src_address, src_stride, best_address, ref_stride);
+ const int skip_sad =
+ ms_params->vfp->sdsf(src_address, src_stride, best_address, ref_stride);
+ // We will keep the result of skipping rows if it's good enough. Here, good
+ // enough means the error is less than 1 per pixel.
+ const int kSADThresh =
+ 1 << (mi_size_wide_log2[bsize] + mi_size_high_log2[bsize]);
+ if (sad > kSADThresh && abs(skip_sad - sad) * 10 >= AOMMAX(sad, 1) * 9) {
+ // There is a large discrepancy between skipping and not skipping, so we
+ // need to redo the motion search.
+ FULLPEL_MOTION_SEARCH_PARAMS new_ms_params = *ms_params;
+ new_ms_params.sdf = new_ms_params.vfp->sdf;
+ new_ms_params.sdx4df = new_ms_params.vfp->sdx4df;
+
+ return av1_full_pixel_search(start_mv, &new_ms_params, step_param,
+ cost_list, best_mv, second_best_mv);
+ }
+ }
+
if (run_mesh_search) {
int var_ex;
FULLPEL_MV tmp_mv_ex;
diff --git a/av1/encoder/mcomp.h b/av1/encoder/mcomp.h
index 05ae381..2519cc8 100644
--- a/av1/encoder/mcomp.h
+++ b/av1/encoder/mcomp.h
@@ -191,6 +191,11 @@
// For calculating mv cost
MV_COST_PARAMS mv_cost_params;
+
+ // Stores the function used to compute the sad. This can be different from the
+ // sdf in vfp (e.g. downsampled sad and not sad) to allow speed up.
+ aom_sad_fn_t sdf;
+ aom_sad_multi_d_fn_t sdx4df;
} FULLPEL_MOTION_SEARCH_PARAMS;
void av1_make_default_fullpel_ms_params(
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index ce4fbee..e287649 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -161,6 +161,12 @@
sf->part_sf.ml_early_term_after_part_split_level = 1;
}
+ if (is_720p_or_larger) {
+ // TODO(chiyotsai@google.com): make this speed feature adaptive based on
+ // current block's vertical texture instead of hardcoded with resolution
+ sf->mv_sf.use_downsampled_sad = 1;
+ }
+
if (speed >= 1) {
if (is_720p_or_larger) {
sf->part_sf.use_square_partition_only_threshold = BLOCK_128X128;
@@ -1016,6 +1022,7 @@
mv_sf->use_accurate_subpel_search = USE_8_TAPS;
mv_sf->use_bsize_dependent_search_method = 0;
mv_sf->use_fullpel_costlist = 0;
+ mv_sf->use_downsampled_sad = 0;
}
static AOM_INLINE void init_inter_sf(INTER_MODE_SPEED_FEATURES *inter_sf) {
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index 1ed4ac6..a59a172 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -533,6 +533,10 @@
// Accurate full pixel motion search based on TPL stats.
int full_pixel_search_level;
+
+ // Whether to downsample the rows in sad calculation during motion search.
+ // This is only active when there are at least 16 rows.
+ int use_downsampled_sad;
} MV_SPEED_FEATURES;
typedef struct INTER_MODE_SPEED_FEATURES {
diff --git a/test/sad_test.cc b/test/sad_test.cc
index 101204f..fdcf49f 100644
--- a/test/sad_test.cc
+++ b/test/sad_test.cc
@@ -31,6 +31,10 @@
const uint8_t *ref_ptr, int ref_stride);
typedef std::tuple<int, int, SadMxNFunc, int> SadMxNParam;
+typedef unsigned int (*SadSkipMxNFunc)(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *ref_ptr, int ref_stride);
+typedef std::tuple<int, int, SadSkipMxNFunc, int> SadSkipMxNParam;
+
typedef uint32_t (*SadMxNAvgFunc)(const uint8_t *src_ptr, int src_stride,
const uint8_t *ref_ptr, int ref_stride,
const uint8_t *second_pred);
@@ -60,6 +64,11 @@
uint32_t *sad_array);
typedef std::tuple<int, int, SadMxNx4Func, int> SadMxNx4Param;
+typedef void (*SadSkipMxNx4Func)(const uint8_t *src_ptr, int src_stride,
+ const uint8_t *const ref_ptr[], int ref_stride,
+ uint32_t *sad_array);
+typedef std::tuple<int, int, SadSkipMxNx4Func, int> SadSkipMxNx4Param;
+
typedef void (*SadMxNx4AvgFunc)(const uint8_t *src_ptr, int src_stride,
const uint8_t *const ref_ptr[], int ref_stride,
const uint8_t *second_pred,
@@ -182,6 +191,31 @@
return sad;
}
+ // Sum of Absolute Differences Skip rows. Given two blocks,
+ // calculate the absolute difference between two pixels in the same
+ // relative location every other row; accumulate and double the result at the
+ // end.
+ unsigned int ReferenceSADSkip(int block_idx) {
+ unsigned int sad = 0;
+ const uint8_t *const reference8 = GetReference(block_idx);
+ const uint8_t *const source8 = source_data_;
+ const uint16_t *const reference16 =
+ CONVERT_TO_SHORTPTR(GetReference(block_idx));
+ const uint16_t *const source16 = CONVERT_TO_SHORTPTR(source_data_);
+ for (int h = 0; h < height_; h += 2) {
+ for (int w = 0; w < width_; ++w) {
+ if (!use_high_bit_depth_) {
+ sad += abs(source8[h * source_stride_ + w] -
+ reference8[h * reference_stride_ + w]);
+ } else {
+ sad += abs(source16[h * source_stride_ + w] -
+ reference16[h * reference_stride_ + w]);
+ }
+ }
+ }
+ return sad * 2;
+ }
+
// Sum of Absolute Differences Average. Given two blocks, and a prediction
// calculate the absolute difference between one pixel and average of the
// corresponding and predicted pixels; accumulate.
@@ -343,6 +377,50 @@
EXPECT_EQ(reference_sad, exp_sad[block]) << "block " << block;
}
}
+
+ void SpeedSAD() {
+ int test_count = 2000000;
+ unsigned int exp_sad[4];
+ while (test_count > 0) {
+ SADs(exp_sad);
+ test_count -= 1;
+ }
+ }
+};
+
+class SADSkipx4Test : public ::testing::WithParamInterface<SadMxNx4Param>,
+ public SADTestBase {
+ public:
+ SADSkipx4Test() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {}
+
+ protected:
+ void SADs(unsigned int *results) {
+ const uint8_t *references[] = { GetReference(0), GetReference(1),
+ GetReference(2), GetReference(3) };
+
+ ASM_REGISTER_STATE_CHECK(GET_PARAM(2)(
+ source_data_, source_stride_, references, reference_stride_, results));
+ }
+
+ void CheckSADs() {
+ unsigned int reference_sad, exp_sad[4];
+
+ SADs(exp_sad);
+ for (int block = 0; block < 4; ++block) {
+ reference_sad = ReferenceSADSkip(block);
+
+ EXPECT_EQ(reference_sad, exp_sad[block]) << "block " << block;
+ }
+ }
+
+ void SpeedSAD() {
+ int test_count = 2000000;
+ unsigned int exp_sad[4];
+ while (test_count > 0) {
+ SADs(exp_sad);
+ test_count -= 1;
+ }
+ }
};
class SADx4AvgTest : public ::testing::WithParamInterface<SadMxNx4AvgParam>,
@@ -412,6 +490,37 @@
}
};
+class SADSkipTest : public ::testing::WithParamInterface<SadMxNParam>,
+ public SADTestBase {
+ public:
+ SADSkipTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {}
+
+ protected:
+ unsigned int SAD(int block_idx) {
+ unsigned int ret;
+ const uint8_t *const reference = GetReference(block_idx);
+
+ ASM_REGISTER_STATE_CHECK(ret = GET_PARAM(2)(source_data_, source_stride_,
+ reference, reference_stride_));
+ return ret;
+ }
+
+ void CheckSAD() {
+ const unsigned int reference_sad = ReferenceSADSkip(0);
+ const unsigned int exp_sad = SAD(0);
+
+ ASSERT_EQ(reference_sad, exp_sad);
+ }
+
+ void SpeedSAD() {
+ int test_count = 20000000;
+ while (test_count > 0) {
+ SAD(0);
+ test_count -= 1;
+ }
+ }
+};
+
class SADavgTest : public ::testing::WithParamInterface<SadMxNAvgParam>,
public SADTestBase {
public:
@@ -608,6 +717,62 @@
}
#endif
+TEST_P(SADSkipTest, MaxRef) {
+ FillConstant(source_data_, source_stride_, 0);
+ FillConstant(reference_data_, reference_stride_, mask_);
+ CheckSAD();
+}
+
+TEST_P(SADSkipTest, MaxSrc) {
+ FillConstant(source_data_, source_stride_, mask_);
+ FillConstant(reference_data_, reference_stride_, 0);
+ CheckSAD();
+}
+
+TEST_P(SADSkipTest, ShortRef) {
+ const int tmp_stride = reference_stride_;
+ reference_stride_ >>= 1;
+ FillRandom(source_data_, source_stride_);
+ FillRandom(reference_data_, reference_stride_);
+ CheckSAD();
+ reference_stride_ = tmp_stride;
+}
+
+TEST_P(SADSkipTest, UnalignedRef) {
+ // The reference frame, but not the source frame, may be unaligned for
+ // certain types of searches.
+ const int tmp_stride = reference_stride_;
+ reference_stride_ -= 1;
+ FillRandom(source_data_, source_stride_);
+ FillRandom(reference_data_, reference_stride_);
+ CheckSAD();
+ reference_stride_ = tmp_stride;
+}
+
+TEST_P(SADSkipTest, ShortSrc) {
+ const int tmp_stride = source_stride_;
+ source_stride_ >>= 1;
+ int test_count = 2000;
+ while (test_count > 0) {
+ FillRandom(source_data_, source_stride_);
+ FillRandom(reference_data_, reference_stride_);
+ CheckSAD();
+ test_count -= 1;
+ }
+ source_stride_ = tmp_stride;
+}
+
+#if SPEED_TEST
+TEST_P(SADSkipTest, Speed) {
+ const int tmp_stride = source_stride_;
+ source_stride_ >>= 1;
+ FillRandom(source_data_, source_stride_);
+ FillRandom(reference_data_, reference_stride_);
+ SpeedSAD();
+ source_stride_ = tmp_stride;
+}
+#endif
+
TEST_P(SADavgTest, MaxRef) {
FillConstant(source_data_, source_stride_, 0);
FillConstant(reference_data_, reference_stride_, mask_);
@@ -855,6 +1020,101 @@
source_data_ = tmp_source_data;
}
+#if SPEED_TEST
+TEST_P(SADx4Test, Speed) {
+ FillRandom(source_data_, source_stride_);
+ FillRandom(GetReference(0), reference_stride_);
+ FillRandom(GetReference(1), reference_stride_);
+ FillRandom(GetReference(2), reference_stride_);
+ FillRandom(GetReference(3), reference_stride_);
+ SpeedSAD();
+}
+#endif
+
+// SADSkipx4
+TEST_P(SADSkipx4Test, MaxRef) {
+ FillConstant(source_data_, source_stride_, 0);
+ FillConstant(GetReference(0), reference_stride_, mask_);
+ FillConstant(GetReference(1), reference_stride_, mask_);
+ FillConstant(GetReference(2), reference_stride_, mask_);
+ FillConstant(GetReference(3), reference_stride_, mask_);
+ CheckSADs();
+}
+
+TEST_P(SADSkipx4Test, MaxSrc) {
+ FillConstant(source_data_, source_stride_, mask_);
+ FillConstant(GetReference(0), reference_stride_, 0);
+ FillConstant(GetReference(1), reference_stride_, 0);
+ FillConstant(GetReference(2), reference_stride_, 0);
+ FillConstant(GetReference(3), reference_stride_, 0);
+ CheckSADs();
+}
+
+TEST_P(SADSkipx4Test, ShortRef) {
+ int tmp_stride = reference_stride_;
+ reference_stride_ >>= 1;
+ FillRandom(source_data_, source_stride_);
+ FillRandom(GetReference(0), reference_stride_);
+ FillRandom(GetReference(1), reference_stride_);
+ FillRandom(GetReference(2), reference_stride_);
+ FillRandom(GetReference(3), reference_stride_);
+ CheckSADs();
+ reference_stride_ = tmp_stride;
+}
+
+TEST_P(SADSkipx4Test, UnalignedRef) {
+ // The reference frame, but not the source frame, may be unaligned for
+ // certain types of searches.
+ int tmp_stride = reference_stride_;
+ reference_stride_ -= 1;
+ FillRandom(source_data_, source_stride_);
+ FillRandom(GetReference(0), reference_stride_);
+ FillRandom(GetReference(1), reference_stride_);
+ FillRandom(GetReference(2), reference_stride_);
+ FillRandom(GetReference(3), reference_stride_);
+ CheckSADs();
+ reference_stride_ = tmp_stride;
+}
+
+TEST_P(SADSkipx4Test, ShortSrc) {
+ int tmp_stride = source_stride_;
+ source_stride_ >>= 1;
+ int test_count = 1000;
+ while (test_count > 0) {
+ FillRandom(source_data_, source_stride_);
+ FillRandom(GetReference(0), reference_stride_);
+ FillRandom(GetReference(1), reference_stride_);
+ FillRandom(GetReference(2), reference_stride_);
+ FillRandom(GetReference(3), reference_stride_);
+ CheckSADs();
+ test_count -= 1;
+ }
+ source_stride_ = tmp_stride;
+}
+
+TEST_P(SADSkipx4Test, SrcAlignedByWidth) {
+ uint8_t *tmp_source_data = source_data_;
+ source_data_ += width_;
+ FillRandom(source_data_, source_stride_);
+ FillRandom(GetReference(0), reference_stride_);
+ FillRandom(GetReference(1), reference_stride_);
+ FillRandom(GetReference(2), reference_stride_);
+ FillRandom(GetReference(3), reference_stride_);
+ CheckSADs();
+ source_data_ = tmp_source_data;
+}
+
+#if SPEED_TEST
+TEST_P(SADSkipx4Test, Speed) {
+ FillRandom(source_data_, source_stride_);
+ FillRandom(GetReference(0), reference_stride_);
+ FillRandom(GetReference(1), reference_stride_);
+ FillRandom(GetReference(2), reference_stride_);
+ FillRandom(GetReference(3), reference_stride_);
+ SpeedSAD();
+}
+#endif
+
using std::make_tuple;
#if SPEED_TEST
@@ -1022,6 +1282,103 @@
};
INSTANTIATE_TEST_SUITE_P(C, SADTest, ::testing::ValuesIn(c_tests));
+const SadSkipMxNParam skip_c_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128_c, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64_c, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128_c, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64_c, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32_c, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64_c, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32_c, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16_c, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32_c, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16_c, -1),
+ make_tuple(16, 8, &aom_sad_skip_16x8_c, -1),
+ make_tuple(8, 16, &aom_sad_skip_8x16_c, -1),
+ make_tuple(8, 8, &aom_sad_skip_8x8_c, -1),
+ make_tuple(8, 4, &aom_sad_skip_8x4_c, -1),
+ make_tuple(4, 8, &aom_sad_skip_4x8_c, -1),
+ make_tuple(4, 4, &aom_sad_skip_4x4_c, -1),
+ make_tuple(64, 16, &aom_sad_skip_64x16_c, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64_c, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8_c, -1),
+ make_tuple(8, 32, &aom_sad_skip_8x32_c, -1),
+ make_tuple(16, 4, &aom_sad_skip_16x4_c, -1),
+ make_tuple(4, 16, &aom_sad_skip_4x16_c, -1),
+
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128_c, 8),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64_c, 8),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128_c, 8),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_c, 8),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_c, 8),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_c, 8),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_c, 8),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_c, 8),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_c, 8),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_c, 8),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_c, 8),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16_c, 8),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8_c, 8),
+ make_tuple(8, 4, &aom_highbd_sad_skip_8x4_c, 8),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8_c, 8),
+ make_tuple(4, 4, &aom_highbd_sad_skip_4x4_c, 8),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16_c, 8),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_c, 8),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8_c, 8),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32_c, 8),
+ make_tuple(16, 4, &aom_highbd_sad_skip_16x4_c, 8),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16_c, 8),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128_c, 10),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64_c, 10),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128_c, 10),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_c, 10),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_c, 10),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_c, 10),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_c, 10),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_c, 10),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_c, 10),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_c, 10),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_c, 10),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16_c, 10),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8_c, 10),
+ make_tuple(8, 4, &aom_highbd_sad_skip_8x4_c, 10),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8_c, 10),
+ make_tuple(4, 4, &aom_highbd_sad_skip_4x4_c, 10),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16_c, 10),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_c, 10),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8_c, 10),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32_c, 10),
+ make_tuple(16, 4, &aom_highbd_sad_skip_16x4_c, 10),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16_c, 10),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128_c, 12),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64_c, 12),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128_c, 12),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_c, 12),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_c, 12),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_c, 12),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_c, 12),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_c, 12),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_c, 12),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_c, 12),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_c, 12),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16_c, 12),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8_c, 12),
+ make_tuple(8, 4, &aom_highbd_sad_skip_8x4_c, 12),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8_c, 12),
+ make_tuple(4, 4, &aom_highbd_sad_skip_4x4_c, 12),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16_c, 12),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_c, 12),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8_c, 12),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32_c, 12),
+ make_tuple(16, 4, &aom_highbd_sad_skip_16x4_c, 12),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16_c, 12),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(C, SADSkipTest, ::testing::ValuesIn(skip_c_tests));
+
const SadMxNAvgParam avg_c_tests[] = {
make_tuple(128, 128, &aom_sad128x128_avg_c, -1),
make_tuple(128, 64, &aom_sad128x64_avg_c, -1),
@@ -1281,6 +1638,101 @@
};
INSTANTIATE_TEST_SUITE_P(C, SADx4Test, ::testing::ValuesIn(x4d_c_tests));
+const SadMxNx4Param skip_x4d_c_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128x4d_c, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64x4d_c, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128x4d_c, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64x4d_c, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32x4d_c, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64x4d_c, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32x4d_c, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16x4d_c, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32x4d_c, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16x4d_c, -1),
+ make_tuple(16, 8, &aom_sad_skip_16x8x4d_c, -1),
+ make_tuple(8, 16, &aom_sad_skip_8x16x4d_c, -1),
+ make_tuple(8, 8, &aom_sad_skip_8x8x4d_c, -1),
+ make_tuple(4, 8, &aom_sad_skip_4x8x4d_c, -1),
+ make_tuple(64, 16, &aom_sad_skip_64x16x4d_c, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64x4d_c, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8x4d_c, -1),
+ make_tuple(8, 32, &aom_sad_skip_8x32x4d_c, -1),
+ make_tuple(4, 16, &aom_sad_skip_4x16x4d_c, -1),
+
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128x4d_c, 8),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64x4d_c, 8),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128x4d_c, 8),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_c, 8),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_c, 8),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_c, 8),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_c, 8),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_c, 8),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_c, 8),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_c, 8),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_c, 8),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16x4d_c, 8),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8x4d_c, 8),
+ make_tuple(8, 4, &aom_highbd_sad_skip_8x4x4d_c, 8),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8x4d_c, 8),
+ make_tuple(4, 4, &aom_highbd_sad_skip_4x4x4d_c, 8),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_c, 8),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_c, 8),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_c, 8),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32x4d_c, 8),
+ make_tuple(16, 4, &aom_highbd_sad_skip_16x4x4d_c, 8),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16x4d_c, 8),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128x4d_c, 10),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64x4d_c, 10),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128x4d_c, 10),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_c, 10),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_c, 10),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_c, 10),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_c, 10),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_c, 10),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_c, 10),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_c, 10),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_c, 10),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16x4d_c, 10),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8x4d_c, 10),
+ make_tuple(8, 4, &aom_highbd_sad_skip_8x4x4d_c, 10),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8x4d_c, 10),
+ make_tuple(4, 4, &aom_highbd_sad_skip_4x4x4d_c, 10),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_c, 10),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_c, 10),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_c, 10),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32x4d_c, 10),
+ make_tuple(16, 4, &aom_highbd_sad_skip_16x4x4d_c, 10),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16x4d_c, 10),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128x4d_c, 12),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64x4d_c, 12),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128x4d_c, 12),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_c, 12),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_c, 12),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_c, 12),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_c, 12),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_c, 12),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_c, 12),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_c, 12),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_c, 12),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16x4d_c, 12),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8x4d_c, 12),
+ make_tuple(8, 4, &aom_highbd_sad_skip_8x4x4d_c, 12),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8x4d_c, 12),
+ make_tuple(4, 4, &aom_highbd_sad_skip_4x4x4d_c, 12),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_c, 12),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_c, 12),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_c, 12),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32x4d_c, 12),
+ make_tuple(16, 4, &aom_highbd_sad_skip_16x4x4d_c, 12),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16x4d_c, 12),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(C, SADSkipx4Test,
+ ::testing::ValuesIn(skip_x4d_c_tests));
+
const SadMxNx4AvgParam x4d_avg_c_tests[] = {
make_tuple(128, 128, &aom_sad128x128x4d_avg_c, -1),
make_tuple(128, 64, &aom_sad128x64x4d_avg_c, -1),
@@ -1424,6 +1876,83 @@
};
INSTANTIATE_TEST_SUITE_P(SSE2, SADTest, ::testing::ValuesIn(sse2_tests));
+const SadSkipMxNParam skip_sse2_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128_sse2, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64_sse2, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128_sse2, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64_sse2, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32_sse2, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64_sse2, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32_sse2, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16_sse2, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32_sse2, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16_sse2, -1),
+ make_tuple(16, 8, &aom_sad_skip_16x8_sse2, -1),
+ make_tuple(8, 16, &aom_sad_skip_8x16_sse2, -1),
+ make_tuple(8, 8, &aom_sad_skip_8x8_sse2, -1),
+ make_tuple(4, 8, &aom_sad_skip_4x8_sse2, -1),
+ make_tuple(64, 16, &aom_sad_skip_64x16_sse2, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64_sse2, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8_sse2, -1),
+ make_tuple(8, 32, &aom_sad_skip_8x32_sse2, -1),
+ make_tuple(4, 16, &aom_sad_skip_4x16_sse2, -1),
+
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_sse2, 8),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_sse2, 8),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_sse2, 8),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_sse2, 8),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_sse2, 8),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_sse2, 8),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_sse2, 8),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_sse2, 8),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16_sse2, 8),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8_sse2, 8),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8_sse2, 8),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16_sse2, 8),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_sse2, 8),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8_sse2, 8),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32_sse2, 8),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16_sse2, 8),
+
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_sse2, 10),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_sse2, 10),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_sse2, 10),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_sse2, 10),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_sse2, 10),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_sse2, 10),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_sse2, 10),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_sse2, 10),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16_sse2, 10),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8_sse2, 10),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8_sse2, 10),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16_sse2, 10),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_sse2, 10),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8_sse2, 10),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32_sse2, 10),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16_sse2, 10),
+
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_sse2, 12),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_sse2, 12),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_sse2, 12),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_sse2, 12),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_sse2, 12),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_sse2, 12),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_sse2, 12),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_sse2, 12),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16_sse2, 12),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8_sse2, 12),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8_sse2, 12),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16_sse2, 12),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_sse2, 12),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8_sse2, 12),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32_sse2, 12),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16_sse2, 12),
+#endif // CONFIG_AV1_HIGHBITDEPTH
+};
+INSTANTIATE_TEST_SUITE_P(SSE2, SADSkipTest,
+ ::testing::ValuesIn(skip_sse2_tests));
+
const SadMxNAvgParam avg_sse2_tests[] = {
make_tuple(128, 128, &aom_sad128x128_avg_sse2, -1),
make_tuple(128, 64, &aom_sad128x64_avg_sse2, -1),
@@ -1606,6 +2135,84 @@
};
INSTANTIATE_TEST_SUITE_P(SSE2, SADx4Test, ::testing::ValuesIn(x4d_sse2_tests));
+const SadSkipMxNx4Param skip_x4d_sse2_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128x4d_sse2, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64x4d_sse2, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128x4d_sse2, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64x4d_sse2, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32x4d_sse2, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64x4d_sse2, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32x4d_sse2, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16x4d_sse2, -1),
+ make_tuple(16, 32, &aom_sad_skip_16x32x4d_sse2, -1),
+ make_tuple(16, 16, &aom_sad_skip_16x16x4d_sse2, -1),
+ make_tuple(16, 8, &aom_sad_skip_16x8x4d_sse2, -1),
+ make_tuple(8, 16, &aom_sad_skip_8x16x4d_sse2, -1),
+ make_tuple(8, 8, &aom_sad_skip_8x8x4d_sse2, -1),
+ make_tuple(4, 8, &aom_sad_skip_4x8x4d_sse2, -1),
+ make_tuple(64, 16, &aom_sad_skip_64x16x4d_sse2, -1),
+ make_tuple(16, 64, &aom_sad_skip_16x64x4d_sse2, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8x4d_sse2, -1),
+ make_tuple(8, 32, &aom_sad_skip_8x32x4d_sse2, -1),
+ make_tuple(4, 16, &aom_sad_skip_4x16x4d_sse2, -1),
+
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_sse2, 8),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_sse2, 8),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_sse2, 8),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_sse2, 8),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_sse2, 8),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_sse2, 8),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_sse2, 8),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_sse2, 8),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16x4d_sse2, 8),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8x4d_sse2, 8),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8x4d_sse2, 8),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_sse2, 8),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_sse2, 8),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_sse2, 8),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32x4d_sse2, 8),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16x4d_sse2, 8),
+
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_sse2, 10),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_sse2, 10),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_sse2, 10),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_sse2, 10),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_sse2, 10),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_sse2, 10),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_sse2, 10),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_sse2, 10),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16x4d_sse2, 10),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8x4d_sse2, 10),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8x4d_sse2, 10),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_sse2, 10),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_sse2, 10),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_sse2, 10),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32x4d_sse2, 10),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16x4d_sse2, 10),
+
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_sse2, 12),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_sse2, 12),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_sse2, 12),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_sse2, 12),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_sse2, 12),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_sse2, 12),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_sse2, 12),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_sse2, 12),
+ make_tuple(8, 16, &aom_highbd_sad_skip_8x16x4d_sse2, 12),
+ make_tuple(8, 8, &aom_highbd_sad_skip_8x8x4d_sse2, 12),
+ make_tuple(4, 8, &aom_highbd_sad_skip_4x8x4d_sse2, 12),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_sse2, 12),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_sse2, 12),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_sse2, 12),
+ make_tuple(8, 32, &aom_highbd_sad_skip_8x32x4d_sse2, 12),
+ make_tuple(4, 16, &aom_highbd_sad_skip_4x16x4d_sse2, 12),
+
+#endif // CONFIG_AV1_HIGHBITDEPTH
+};
+INSTANTIATE_TEST_SUITE_P(SSE2, SADSkipx4Test,
+ ::testing::ValuesIn(skip_x4d_sse2_tests));
+
const SadMxNx4AvgParam x4d_avg_sse2_tests[] = {
make_tuple(128, 128, &aom_sad128x128x4d_avg_sse2, -1),
make_tuple(128, 64, &aom_sad128x64x4d_avg_sse2, -1),
@@ -1802,6 +2409,59 @@
};
INSTANTIATE_TEST_SUITE_P(AVX2, SADTest, ::testing::ValuesIn(avx2_tests));
+const SadSkipMxNParam skip_avx2_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128_avx2, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64_avx2, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128_avx2, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64_avx2, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32_avx2, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64_avx2, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32_avx2, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16_avx2, -1),
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128_avx2, 8),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64_avx2, 8),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128_avx2, 8),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_avx2, 8),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_avx2, 8),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_avx2, 8),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_avx2, 8),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_avx2, 8),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_avx2, 8),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_avx2, 8),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_avx2, 8),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_avx2, 8),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128_avx2, 10),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64_avx2, 10),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128_avx2, 10),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_avx2, 10),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_avx2, 10),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_avx2, 10),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_avx2, 10),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_avx2, 10),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_avx2, 10),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_avx2, 10),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_avx2, 10),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_avx2, 10),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128_avx2, 12),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64_avx2, 12),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128_avx2, 12),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64_avx2, 12),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32_avx2, 12),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64_avx2, 12),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32_avx2, 12),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16_avx2, 12),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64_avx2, 12),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32_avx2, 12),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16_avx2, 12),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8_avx2, 12),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(AVX2, SADSkipTest,
+ ::testing::ValuesIn(skip_avx2_tests));
+
const SadMxNAvgParam avg_avx2_tests[] = {
make_tuple(64, 128, &aom_sad64x128_avg_avx2, -1),
make_tuple(128, 64, &aom_sad128x64_avg_avx2, -1),
@@ -1862,6 +2522,67 @@
};
INSTANTIATE_TEST_SUITE_P(AVX2, SADavgTest, ::testing::ValuesIn(avg_avx2_tests));
+const SadSkipMxNx4Param skip_x4d_avx2_tests[] = {
+ make_tuple(128, 128, &aom_sad_skip_128x128x4d_avx2, -1),
+ make_tuple(128, 64, &aom_sad_skip_128x64x4d_avx2, -1),
+ make_tuple(64, 128, &aom_sad_skip_64x128x4d_avx2, -1),
+ make_tuple(64, 64, &aom_sad_skip_64x64x4d_avx2, -1),
+ make_tuple(64, 32, &aom_sad_skip_64x32x4d_avx2, -1),
+ make_tuple(64, 16, &aom_sad_skip_64x16x4d_avx2, -1),
+ make_tuple(32, 64, &aom_sad_skip_32x64x4d_avx2, -1),
+ make_tuple(32, 32, &aom_sad_skip_32x32x4d_avx2, -1),
+ make_tuple(32, 16, &aom_sad_skip_32x16x4d_avx2, -1),
+ make_tuple(32, 8, &aom_sad_skip_32x8x4d_avx2, -1),
+#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128x4d_avx2, 8),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64x4d_avx2, 8),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128x4d_avx2, 8),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_avx2, 8),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_avx2, 8),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_avx2, 8),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_avx2, 8),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_avx2, 8),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_avx2, 8),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_avx2, 8),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_avx2, 8),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_avx2, 8),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_avx2, 8),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_avx2, 8),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128x4d_avx2, 10),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64x4d_avx2, 10),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128x4d_avx2, 10),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_avx2, 10),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_avx2, 10),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_avx2, 10),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_avx2, 10),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_avx2, 10),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_avx2, 10),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_avx2, 10),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_avx2, 10),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_avx2, 10),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_avx2, 10),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_avx2, 10),
+
+ make_tuple(128, 128, &aom_highbd_sad_skip_128x128x4d_avx2, 12),
+ make_tuple(128, 64, &aom_highbd_sad_skip_128x64x4d_avx2, 12),
+ make_tuple(64, 128, &aom_highbd_sad_skip_64x128x4d_avx2, 12),
+ make_tuple(64, 64, &aom_highbd_sad_skip_64x64x4d_avx2, 12),
+ make_tuple(64, 32, &aom_highbd_sad_skip_64x32x4d_avx2, 12),
+ make_tuple(64, 16, &aom_highbd_sad_skip_64x16x4d_avx2, 12),
+ make_tuple(32, 64, &aom_highbd_sad_skip_32x64x4d_avx2, 12),
+ make_tuple(32, 32, &aom_highbd_sad_skip_32x32x4d_avx2, 12),
+ make_tuple(32, 16, &aom_highbd_sad_skip_32x16x4d_avx2, 12),
+ make_tuple(32, 8, &aom_highbd_sad_skip_32x8x4d_avx2, 12),
+ make_tuple(16, 64, &aom_highbd_sad_skip_16x64x4d_avx2, 12),
+ make_tuple(16, 32, &aom_highbd_sad_skip_16x32x4d_avx2, 12),
+ make_tuple(16, 16, &aom_highbd_sad_skip_16x16x4d_avx2, 12),
+ make_tuple(16, 8, &aom_highbd_sad_skip_16x8x4d_avx2, 12),
+#endif
+};
+INSTANTIATE_TEST_SUITE_P(AVX2, SADSkipx4Test,
+ ::testing::ValuesIn(skip_x4d_avx2_tests));
+
const SadMxNx4Param x4d_avx2_tests[] = {
make_tuple(32, 64, &aom_sad32x64x4d_avx2, -1),
make_tuple(32, 32, &aom_sad32x32x4d_avx2, -1),
