Add experiment CONFIG_CDEF_SINGLEPASS: Make CDEF single pass
Low latency, cpu-used=0:
PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000
-0.3162 | -0.6719 | -0.6535 | 0.0089 | -0.3890 | -0.1515 | -0.6682
High latency, cpu-used=0:
PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000
-0.0293 | -0.3556 | -0.5505 | 0.0684 | -0.0862 | 0.0513 | -0.2765
Low latency, cpu-used=4:
PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000
-0.2248 | -0.7764 | -0.6630 | -0.2109 | -0.3240 | -0.2532 | -0.6980
High latency, cpu-used=4:
PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000
-0.1118 | -0.5841 | -0.7406 | -0.0463 | -0.2442 | -0.1064 | -0.4187
Change-Id: I9ca8399c8f45489541a66f535fb3d771eb1d59ab
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 46e55bb..e7d7e10 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -236,13 +236,15 @@
if (CONFIG_CDEF)
set(AOM_AV1_COMMON_SOURCES
${AOM_AV1_COMMON_SOURCES}
- "${AOM_ROOT}/av1/common/clpf.c"
- "${AOM_ROOT}/av1/common/clpf_simd.h"
+ if (!CONFIG_CDEF_SINGLEPASS)
+ "${AOM_ROOT}/av1/common/clpf.c"
+ "${AOM_ROOT}/av1/common/clpf_simd.h"
+ "${AOM_ROOT}/av1/common/cdef_block_simd.h")
+ endif ()
"${AOM_ROOT}/av1/common/cdef.c"
"${AOM_ROOT}/av1/common/cdef.h"
"${AOM_ROOT}/av1/common/cdef_block.c"
- "${AOM_ROOT}/av1/common/cdef_block.h"
- "${AOM_ROOT}/av1/common/cdef_block_simd.h")
+ "${AOM_ROOT}/av1/common/cdef_block.h")
set(AOM_AV1_ENCODER_SOURCES
${AOM_AV1_ENCODER_SOURCES}
@@ -250,22 +252,34 @@
set(AOM_AV1_COMMON_INTRIN_SSE2
${AOM_AV1_COMMON_INTRIN_SSE2}
- "${AOM_ROOT}/av1/common/clpf_sse2.c"
+ if (!CONFIG_CDEF_SINGLEPASS)
+ "${AOM_ROOT}/av1/common/clpf_sse2.c"
+ endif ()
"${AOM_ROOT}/av1/common/cdef_block_sse2.c")
set(AOM_AV1_COMMON_INTRIN_SSSE3
${AOM_AV1_COMMON_INTRIN_SSSE3}
- "${AOM_ROOT}/av1/common/clpf_ssse3.c"
+ if (!CONFIG_CDEF_SINGLEPASS)
+ "${AOM_ROOT}/av1/common/clpf_ssse3.c"
+ endif ()
"${AOM_ROOT}/av1/common/cdef_block_ssse3.c")
set(AOM_AV1_COMMON_INTRIN_SSE4_1
${AOM_AV1_COMMON_INTRIN_SSE4_1}
- "${AOM_ROOT}/av1/common/clpf_sse4.c"
+ if (!CONFIG_CDEF_SINGLEPASS)
+ "${AOM_ROOT}/av1/common/clpf_sse4.c"
+ endif ()
"${AOM_ROOT}/av1/common/cdef_block_sse4.c")
+ set(AOM_AV1_COMMON_INTRIN_AVX2
+ ${AOM_AV1_COMMON_INTRIN_AVX2}
+ "${AOM_ROOT}/av1/common/cdef_block_avx2.c")
+
set(AOM_AV1_COMMON_INTRIN_NEON
${AOM_AV1_COMMON_INTRIN_NEON}
- "${AOM_ROOT}/av1/common/clpf_neon.c"
+ if (!CONFIG_CDEF_SINGLEPASS)
+ "${AOM_ROOT}/av1/common/clpf_neon.c"
+ endif ()
"${AOM_ROOT}/av1/common/cdef_block_neon.c")
endif ()
diff --git a/av1/av1_common.mk b/av1/av1_common.mk
index 5411229..d5d40af 100644
--- a/av1/av1_common.mk
+++ b/av1/av1_common.mk
@@ -90,12 +90,16 @@
AV1_COMMON_SRCS-yes += common/warped_motion.c
endif
ifeq ($(CONFIG_CDEF),yes)
+ifeq ($(CONFIG_CDEF_SINGLEPASS),yes)
+AV1_COMMON_SRCS-$(HAVE_AVX2) += common/cdef_block_avx2.c
+else
AV1_COMMON_SRCS-yes += common/clpf.c
AV1_COMMON_SRCS-yes += common/clpf_simd.h
AV1_COMMON_SRCS-$(HAVE_SSE2) += common/clpf_sse2.c
AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/clpf_ssse3.c
AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/clpf_sse4.c
AV1_COMMON_SRCS-$(HAVE_NEON) += common/clpf_neon.c
+endif
AV1_COMMON_SRCS-$(HAVE_SSE2) += common/cdef_block_sse2.c
AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/cdef_block_ssse3.c
AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/cdef_block_sse4.c
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index edc07f8..dc6a844 100755
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -520,18 +520,22 @@
# Deringing Functions
if (aom_config("CONFIG_CDEF") eq "yes") {
- add_proto qw/void aom_clpf_block_hbd/, "uint16_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
- add_proto qw/void aom_clpf_hblock_hbd/, "uint16_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
- add_proto qw/void aom_clpf_block/, "uint8_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
- add_proto qw/void aom_clpf_hblock/, "uint8_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
add_proto qw/int cdef_find_dir/, "const uint16_t *img, int stride, int32_t *var, int coeff_shift";
- add_proto qw/void cdef_direction_4x4/, "uint16_t *y, int ystride, const uint16_t *in, int threshold, int dir, int damping";
- add_proto qw/void cdef_direction_8x8/, "uint16_t *y, int ystride, const uint16_t *in, int threshold, int dir, int damping";
+ if (aom_config("CONFIG_CDEF_SINGLEPASS") ne "yes") {
+ add_proto qw/void aom_clpf_block_hbd/, "uint16_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
+ add_proto qw/void aom_clpf_hblock_hbd/, "uint16_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
+ add_proto qw/void aom_clpf_block/, "uint8_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
+ add_proto qw/void aom_clpf_hblock/, "uint8_t *dst, const uint16_t *src, int dstride, int sstride, int sizex, int sizey, unsigned int strength, unsigned int bd";
+ add_proto qw/void cdef_direction_4x4/, "uint16_t *y, int ystride, const uint16_t *in, int threshold, int dir, int damping";
+ add_proto qw/void cdef_direction_8x8/, "uint16_t *y, int ystride, const uint16_t *in, int threshold, int dir, int damping";
+ add_proto qw/void copy_8x8_16bit_to_8bit/, "uint8_t *dst, int dstride, const uint16_t *src, int sstride";
+ add_proto qw/void copy_4x4_16bit_to_8bit/, "uint8_t *dst, int dstride, const uint16_t *src, int sstride";
+ add_proto qw/void copy_8x8_16bit_to_16bit/, "uint16_t *dst, int dstride, const uint16_t *src, int sstride";
+ add_proto qw/void copy_4x4_16bit_to_16bit/, "uint16_t *dst, int dstride, const uint16_t *src, int sstride";
+ } else {
+ add_proto qw/void cdef_filter_block/, "uint8_t *dst8, uint16_t *dst16, int dstride, const uint16_t *in, int pri_strength, int sec_strength, int dir, int pri_damping, int sec_damping, int bsize, int max";
+ }
- add_proto qw/void copy_8x8_16bit_to_8bit/, "uint8_t *dst, int dstride, const uint16_t *src, int sstride";
- add_proto qw/void copy_4x4_16bit_to_8bit/, "uint8_t *dst, int dstride, const uint16_t *src, int sstride";
- add_proto qw/void copy_8x8_16bit_to_16bit/, "uint16_t *dst, int dstride, const uint16_t *src, int sstride";
- add_proto qw/void copy_4x4_16bit_to_16bit/, "uint16_t *dst, int dstride, const uint16_t *src, int sstride";
add_proto qw/void copy_rect8_8bit_to_16bit/, "uint16_t *dst, int dstride, const uint8_t *src, int sstride, int v, int h";
add_proto qw/void copy_rect8_16bit_to_16bit/, "uint16_t *dst, int dstride, const uint16_t *src, int sstride, int v, int h";
@@ -539,20 +543,28 @@
# structs as arguments, which makes the v256 type of the intrinsics
# hard to support, so optimizations for this target are disabled.
if ($opts{config} !~ /libs-x86-win32-vs.*/) {
- specialize qw/aom_clpf_block_hbd sse2 ssse3 sse4_1 neon/;
- specialize qw/aom_clpf_hblock_hbd sse2 ssse3 sse4_1 neon/;
- specialize qw/aom_clpf_block sse2 ssse3 sse4_1 neon/;
- specialize qw/aom_clpf_hblock sse2 ssse3 sse4_1 neon/;
- specialize qw/cdef_find_dir sse2 ssse3 sse4_1 neon/;
- specialize qw/cdef_direction_4x4 sse2 ssse3 sse4_1 neon/;
- specialize qw/cdef_direction_8x8 sse2 ssse3 sse4_1 neon/;
+ if (aom_config("CONFIG_CDEF_SINGLEPASS") eq "yes") {
+ specialize qw/cdef_find_dir sse2 ssse3 sse4_1 avx2 neon/;
+ specialize qw/cdef_filter_block sse2 ssse3 sse4_1 avx2 neon/;
+ specialize qw/copy_rect8_8bit_to_16bit sse2 ssse3 sse4_1 avx2 neon/;
+ specialize qw/copy_rect8_16bit_to_16bit sse2 ssse3 sse4_1 avx2 neon/;
+ } else {
+ specialize qw/cdef_find_dir sse2 ssse3 sse4_1 neon/;
+ specialize qw/aom_clpf_block_hbd sse2 ssse3 sse4_1 neon/;
+ specialize qw/aom_clpf_hblock_hbd sse2 ssse3 sse4_1 neon/;
+ specialize qw/aom_clpf_block sse2 ssse3 sse4_1 neon/;
+ specialize qw/aom_clpf_hblock sse2 ssse3 sse4_1 neon/;
+ specialize qw/cdef_find_dir sse2 ssse3 sse4_1 neon/;
+ specialize qw/cdef_direction_4x4 sse2 ssse3 sse4_1 neon/;
+ specialize qw/cdef_direction_8x8 sse2 ssse3 sse4_1 neon/;
- specialize qw/copy_8x8_16bit_to_8bit sse2 ssse3 sse4_1 neon/;
- specialize qw/copy_4x4_16bit_to_8bit sse2 ssse3 sse4_1 neon/;
- specialize qw/copy_8x8_16bit_to_16bit sse2 ssse3 sse4_1 neon/;
- specialize qw/copy_4x4_16bit_to_16bit sse2 ssse3 sse4_1 neon/;
- specialize qw/copy_rect8_8bit_to_16bit sse2 ssse3 sse4_1 neon/;
- specialize qw/copy_rect8_16bit_to_16bit sse2 ssse3 sse4_1 neon/;
+ specialize qw/copy_8x8_16bit_to_8bit sse2 ssse3 sse4_1 neon/;
+ specialize qw/copy_4x4_16bit_to_8bit sse2 ssse3 sse4_1 neon/;
+ specialize qw/copy_8x8_16bit_to_16bit sse2 ssse3 sse4_1 neon/;
+ specialize qw/copy_4x4_16bit_to_16bit sse2 ssse3 sse4_1 neon/;
+ specialize qw/copy_rect8_8bit_to_16bit sse2 ssse3 sse4_1 neon/;
+ specialize qw/copy_rect8_16bit_to_16bit sse2 ssse3 sse4_1 neon/;
+ }
}
}
diff --git a/av1/common/cdef.c b/av1/common/cdef.c
index 8bb3874..8417fea 100644
--- a/av1/common/cdef.c
+++ b/av1/common/cdef.c
@@ -260,14 +260,21 @@
uv_sec_strength == 0) ||
(cdef_count = sb_compute_cdef_list(
cm, fbr * MI_SIZE_64X64, fbc * MI_SIZE_64X64, dlist,
- get_filter_skip(level) || get_filter_skip(uv_level))) == 0) {
+#if CONFIG_CDEF_SINGLEPASS
+ (level & 1) || (uv_level & 1))) == 0)
+#else
+ get_filter_skip(level) || get_filter_skip(uv_level))) == 0)
+#endif
+ {
cdef_left = 0;
continue;
}
curr_row_cdef[fbc] = 1;
for (pli = 0; pli < nplanes; pli++) {
+#if !CONFIG_CDEF_SINGLEPASS
uint16_t dst[CDEF_BLOCKSIZE * CDEF_BLOCKSIZE];
+#endif
int coffset;
int rend, cend;
int pri_damping = cm->cdef_pri_damping;
@@ -386,15 +393,28 @@
#if CONFIG_HIGHBITDEPTH
if (cm->use_highbitdepth) {
cdef_filter_fb(
+#if CONFIG_CDEF_SINGLEPASS
+ NULL,
+ &CONVERT_TO_SHORTPTR(
+#else
(uint8_t *)&CONVERT_TO_SHORTPTR(
+#endif
xd->plane[pli]
.dst.buf)[xd->plane[pli].dst.stride *
(MI_SIZE_64X64 * fbr << mi_high_l2[pli]) +
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli])],
+#if CONFIG_CDEF_SINGLEPASS
+ xd->plane[pli].dst.stride,
+#else
xd->plane[pli].dst.stride, dst,
+#endif
&src[CDEF_VBORDER * CDEF_BSTRIDE + CDEF_HBORDER], xdec[pli],
ydec[pli], dir, NULL, var, pli, dlist, cdef_count, level,
+#if CONFIG_CDEF_SINGLEPASS
+ sec_strength, pri_damping, sec_damping, coeff_shift);
+#else
sec_strength, sec_damping, pri_damping, coeff_shift, 0, 1);
+#endif
} else {
#endif
cdef_filter_fb(
@@ -402,10 +422,18 @@
.dst.buf[xd->plane[pli].dst.stride *
(MI_SIZE_64X64 * fbr << mi_high_l2[pli]) +
(fbc * MI_SIZE_64X64 << mi_wide_l2[pli])],
+#if CONFIG_CDEF_SINGLEPASS
+ NULL, xd->plane[pli].dst.stride,
+#else
xd->plane[pli].dst.stride, dst,
+#endif
&src[CDEF_VBORDER * CDEF_BSTRIDE + CDEF_HBORDER], xdec[pli],
ydec[pli], dir, NULL, var, pli, dlist, cdef_count, level,
+#if CONFIG_CDEF_SINGLEPASS
+ sec_strength, pri_damping, sec_damping, coeff_shift);
+#else
sec_strength, sec_damping, pri_damping, coeff_shift, 0, 0);
+#endif
#if CONFIG_HIGHBITDEPTH
}
diff --git a/av1/common/cdef_block.c b/av1/common/cdef_block.c
index 3fe836a..874644a 100644
--- a/av1/common/cdef_block.c
+++ b/av1/common/cdef_block.c
@@ -21,6 +21,7 @@
#include "./cdef.h"
/* Generated from gen_filter_tables.c. */
+#if !CONFIG_CDEF_SINGLEPASS || CDEF_FULL
const int cdef_directions[8][3] = {
{ -1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2, -3 * CDEF_BSTRIDE + 3 },
{ 0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2, -1 * CDEF_BSTRIDE + 3 },
@@ -31,6 +32,18 @@
{ 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0, 3 * CDEF_BSTRIDE + 0 },
{ 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1, 3 * CDEF_BSTRIDE - 1 }
};
+#else
+const int cdef_directions[8][2] = {
+ { -1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2 },
+ { 0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2 },
+ { 0 * CDEF_BSTRIDE + 1, 0 * CDEF_BSTRIDE + 2 },
+ { 0 * CDEF_BSTRIDE + 1, 1 * CDEF_BSTRIDE + 2 },
+ { 1 * CDEF_BSTRIDE + 1, 2 * CDEF_BSTRIDE + 2 },
+ { 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 1 },
+ { 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0 },
+ { 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1 }
+};
+#endif
/* Detect direction. 0 means 45-degree up-right, 2 is horizontal, and so on.
The search minimizes the weighted variance along all the lines in a
@@ -110,6 +123,94 @@
return best_dir;
}
+#if CONFIG_CDEF_SINGLEPASS
+#if CDEF_FULL
+const int cdef_pri_taps[2][3] = { { 3, 2, 1 }, { 2, 2, 2 } };
+const int cdef_sec_taps[2][2] = { { 3, 1 }, { 3, 1 } };
+#else
+const int cdef_pri_taps[2][2] = { { 4, 2 }, { 3, 3 } };
+const int cdef_sec_taps[2][2] = { { 2, 1 }, { 2, 1 } };
+#endif
+
+/* Smooth in the direction detected. */
+#if CDEF_CAP
+void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride,
+ const uint16_t *in, int pri_strength, int sec_strength,
+ int dir, int pri_damping, int sec_damping, int bsize,
+ UNUSED int max_unused)
+#else
+void cdef_filter_block_c(uint8_t *dst8, uint16_t *dst16, int dstride,
+ const uint16_t *in, int pri_strength, int sec_strength,
+ int dir, int pri_damping, int sec_damping, int bsize,
+ int max)
+#endif
+{
+ int i, j, k;
+ const int s = CDEF_BSTRIDE;
+ const int *pri_taps = cdef_pri_taps[pri_strength & 1];
+ const int *sec_taps = cdef_sec_taps[pri_strength & 1];
+ for (i = 0; i < 4 << (bsize == BLOCK_8X8); i++) {
+ for (j = 0; j < 4 << (bsize == BLOCK_8X8); j++) {
+ int16_t sum = 0;
+ int16_t y;
+ int16_t x = in[i * s + j];
+#if CDEF_CAP
+ int max = x;
+ int min = x;
+#endif
+#if CDEF_FULL
+ for (k = 0; k < 3; k++)
+#else
+ for (k = 0; k < 2; k++)
+#endif
+ {
+ int16_t p0 = in[i * s + j + cdef_directions[dir][k]];
+ int16_t p1 = in[i * s + j - cdef_directions[dir][k]];
+ sum += pri_taps[k] * constrain(p0 - x, pri_strength, pri_damping);
+ sum += pri_taps[k] * constrain(p1 - x, pri_strength, pri_damping);
+#if CDEF_CAP
+ if (p0 != CDEF_VERY_LARGE) max = AOMMAX(p0, max);
+ if (p1 != CDEF_VERY_LARGE) max = AOMMAX(p1, max);
+ min = AOMMIN(p0, min);
+ min = AOMMIN(p1, min);
+#endif
+#if CDEF_FULL
+ if (k == 2) continue;
+#endif
+ int16_t s0 = in[i * s + j + cdef_directions[(dir + 2) & 7][k]];
+ int16_t s1 = in[i * s + j - cdef_directions[(dir + 2) & 7][k]];
+ int16_t s2 = in[i * s + j + cdef_directions[(dir + 6) & 7][k]];
+ int16_t s3 = in[i * s + j - cdef_directions[(dir + 6) & 7][k]];
+#if CDEF_CAP
+ if (s0 != CDEF_VERY_LARGE) max = AOMMAX(s0, max);
+ if (s1 != CDEF_VERY_LARGE) max = AOMMAX(s1, max);
+ if (s2 != CDEF_VERY_LARGE) max = AOMMAX(s2, max);
+ if (s3 != CDEF_VERY_LARGE) max = AOMMAX(s3, max);
+ min = AOMMIN(s0, min);
+ min = AOMMIN(s1, min);
+ min = AOMMIN(s2, min);
+ min = AOMMIN(s3, min);
+#endif
+ sum += sec_taps[k] * constrain(s0 - x, sec_strength, sec_damping);
+ sum += sec_taps[k] * constrain(s1 - x, sec_strength, sec_damping);
+ sum += sec_taps[k] * constrain(s2 - x, sec_strength, sec_damping);
+ sum += sec_taps[k] * constrain(s3 - x, sec_strength, sec_damping);
+ }
+#if CDEF_CAP
+ y = clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
+#else
+ y = clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), 0, max);
+#endif
+ if (dst8)
+ dst8[i * dstride + j] = (uint8_t)y;
+ else
+ dst16[i * dstride + j] = (uint16_t)y;
+ }
+ }
+}
+
+#else
+
/* Smooth in the direction detected. */
void cdef_direction_8x8_c(uint16_t *y, int ystride, const uint16_t *in,
int threshold, int dir, int damping) {
@@ -167,6 +268,7 @@
}
}
}
+#endif
/* Compute the primary filter strength for an 8x8 block based on the
directional variance difference. A high variance difference means
@@ -180,6 +282,7 @@
return var ? (strength * (4 + i) + 8) >> 4 : 0;
}
+#if !CONFIG_CDEF_SINGLEPASS
void copy_8x8_16bit_to_16bit_c(uint16_t *dst, int dstride, const uint16_t *src,
int sstride) {
int i, j;
@@ -303,25 +406,56 @@
cdef_list *dlist, int cdef_count, int level,
int sec_strength, int sec_damping, int pri_damping,
int coeff_shift, int skip_dering, int hbd) {
+#else
+
+void cdef_filter_fb(uint8_t *dst8, uint16_t *dst16, int dstride, uint16_t *in,
+ int xdec, int ydec, int dir[CDEF_NBLOCKS][CDEF_NBLOCKS],
+ int *dirinit, int var[CDEF_NBLOCKS][CDEF_NBLOCKS], int pli,
+ cdef_list *dlist, int cdef_count, int level,
+ int sec_strength, int pri_damping, int sec_damping,
+ int coeff_shift) {
+#endif
int bi;
int bx;
int by;
int bsize, bsizex, bsizey;
+#if CONFIG_CDEF_SINGLEPASS
+ int pri_strength = (level >> 1) << coeff_shift;
+ int filter_skip = level & 1;
+ if (!pri_strength && !sec_strength && filter_skip) {
+ pri_strength = 19 << coeff_shift;
+ sec_strength = 7 << coeff_shift;
+ }
+#else
int threshold = (level >> 1) << coeff_shift;
int filter_skip = get_filter_skip(level);
if (level == 1) threshold = 31 << coeff_shift;
cdef_direction_func cdef_direction[] = { cdef_direction_4x4,
cdef_direction_8x8 };
+#endif
sec_damping += coeff_shift - (pli != AOM_PLANE_Y);
pri_damping += coeff_shift - (pli != AOM_PLANE_Y);
bsize =
ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
bsizex = 3 - xdec;
bsizey = 3 - ydec;
-
- if (!skip_dering) {
+#if CONFIG_CDEF_SINGLEPASS
+ if (dirinit && pri_strength == 0 && sec_strength == 0)
+#else
+ if (!skip_dering)
+#endif
+ {
+#if CONFIG_CDEF_SINGLEPASS
+ // If we're here, both primary and secondary strengths are 0, and
+ // we still haven't written anything to y[] yet, so we just copy
+ // the input to y[]. This is necessary only for av1_cdef_search()
+ // and only av1_cdef_search() sets dirinit.
+ for (bi = 0; bi < cdef_count; bi++) {
+ by = dlist[bi].by;
+ bx = dlist[bi].bx;
+#else
if (pli == 0) {
if (!dirinit || !*dirinit) {
for (bi = 0; bi < cdef_count; bi++) {
@@ -394,12 +528,56 @@
for (bi = 0; bi < cdef_count; bi++) {
by = dlist[bi].by;
bx = dlist[bi].bx;
+#endif
int iy, ix;
// TODO(stemidts/jmvalin): SIMD optimisations
for (iy = 0; iy < 1 << bsizey; iy++)
for (ix = 0; ix < 1 << bsizex; ix++)
+#if CONFIG_CDEF_SINGLEPASS
+ dst16[(bi << (bsizex + bsizey)) + (iy << bsizex) + ix] =
+#else
y[(bi << (bsizex + bsizey)) + (iy << bsizex) + ix] =
+#endif
in[((by << bsizey) + iy) * CDEF_BSTRIDE + (bx << bsizex) + ix];
}
+#if CONFIG_CDEF_SINGLEPASS
+ return;
+#endif
}
+
+#if CONFIG_CDEF_SINGLEPASS
+ if (pli == 0) {
+ if (!dirinit || !*dirinit) {
+ for (bi = 0; bi < cdef_count; bi++) {
+ by = dlist[bi].by;
+ bx = dlist[bi].bx;
+ dir[by][bx] = cdef_find_dir(&in[8 * by * CDEF_BSTRIDE + 8 * bx],
+ CDEF_BSTRIDE, &var[by][bx], coeff_shift);
+ }
+ if (dirinit) *dirinit = 1;
+ }
+ }
+
+ assert(bsize == BLOCK_8X8 || bsize == BLOCK_4X4);
+ for (bi = 0; bi < cdef_count; bi++) {
+ int t = !filter_skip && dlist[bi].skip ? 0 : pri_strength;
+ int s = !filter_skip && dlist[bi].skip ? 0 : sec_strength;
+ by = dlist[bi].by;
+ bx = dlist[bi].bx;
+ if (dst8)
+ cdef_filter_block(
+ &dst8[(by << bsizey) * dstride + (bx << bsizex)], NULL, dstride,
+ &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)],
+ (pli ? t : adjust_strength(t, var[by][bx])), s, t ? dir[by][bx] : 0,
+ pri_damping, sec_damping, bsize, (256 << coeff_shift) - 1);
+ else
+ cdef_filter_block(
+ NULL, &dst16[dirinit ? bi << (bsizex + bsizey)
+ : (by << bsizey) * dstride + (bx << bsizex)],
+ dirinit ? 1 << bsizex : dstride,
+ &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)],
+ (pli ? t : adjust_strength(t, var[by][bx])), s, t ? dir[by][bx] : 0,
+ pri_damping, sec_damping, bsize, (256 << coeff_shift) - 1);
+ }
+#endif
}
diff --git a/av1/common/cdef_block.h b/av1/common/cdef_block.h
index 3891e2b..bf277fa 100644
--- a/av1/common/cdef_block.h
+++ b/av1/common/cdef_block.h
@@ -17,6 +17,9 @@
#define CDEF_BLOCKSIZE 64
#define CDEF_BLOCKSIZE_LOG2 6
#define CDEF_NBLOCKS (CDEF_BLOCKSIZE / 8)
+#if CONFIG_CDEF_SINGLEPASS
+#define CDEF_SB_SHIFT (MAX_SB_SIZE_LOG2 - CDEF_BLOCKSIZE_LOG2)
+#endif
/* We need to buffer three vertical lines. */
#define CDEF_VBORDER (3)
@@ -28,7 +31,24 @@
#define CDEF_VERY_LARGE (30000)
#define CDEF_INBUF_SIZE (CDEF_BSTRIDE * (CDEF_BLOCKSIZE + 2 * CDEF_VBORDER))
+#if CONFIG_CDEF_SINGLEPASS
+// Filter configuration
+#define CDEF_CAP 1 // 1 = Cap change to largest diff
+#define CDEF_FULL 0 // 1 = 7x7 filter, 0 = 5x5 filter
+
+#if CDEF_FULL
+extern const int cdef_pri_taps[2][3];
+extern const int cdef_sec_taps[2][2];
extern const int cdef_directions[8][3];
+#else
+extern const int cdef_pri_taps[2][2];
+extern const int cdef_sec_taps[2][2];
+extern const int cdef_directions[8][2];
+#endif
+
+#else // CONFIG_CDEF_SINGLEPASS
+extern const int cdef_directions[8][3];
+#endif
typedef struct {
uint8_t by;
@@ -36,12 +56,30 @@
uint8_t skip;
} cdef_list;
+#if CONFIG_CDEF_SINGLEPASS
+typedef void (*cdef_filter_block_func)(uint8_t *dst8, uint16_t *dst16,
+ int dstride, const uint16_t *in,
+ int pri_strength, int sec_strength,
+ int dir, int pri_damping,
+ int sec_damping, int bsize, int max);
+void copy_cdef_16bit_to_16bit(uint16_t *dst, int dstride, uint16_t *src,
+ cdef_list *dlist, int cdef_count, int bsize);
+#else
typedef void (*cdef_direction_func)(uint16_t *y, int ystride,
const uint16_t *in, int threshold, int dir,
int damping);
int get_filter_skip(int level);
+#endif
+#if CONFIG_CDEF_SINGLEPASS
+void cdef_filter_fb(uint8_t *dst8, uint16_t *dst16, int dstride, uint16_t *in,
+ int xdec, int ydec, int dir[CDEF_NBLOCKS][CDEF_NBLOCKS],
+ int *dirinit, int var[CDEF_NBLOCKS][CDEF_NBLOCKS], int pli,
+ cdef_list *dlist, int cdef_count, int level,
+ int sec_strength, int pri_damping, int sec_damping,
+ int coeff_shift);
+#else
void cdef_filter_fb(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in,
int xdec, int ydec, int dir[CDEF_NBLOCKS][CDEF_NBLOCKS],
int *dirinit, int var[CDEF_NBLOCKS][CDEF_NBLOCKS], int pli,
@@ -49,3 +87,4 @@
int sec_strength, int sec_damping, int pri_damping,
int coeff_shift, int skip_dering, int hbd);
#endif
+#endif
diff --git a/av1/common/cdef_block_avx2.c b/av1/common/cdef_block_avx2.c
new file mode 100644
index 0000000..5e48045
--- /dev/null
+++ b/av1/common/cdef_block_avx2.c
@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2016, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "aom_dsp/aom_simd.h"
+#define SIMD_FUNC(name) name##_avx2
+#include "./cdef_block_simd.h"
diff --git a/av1/common/cdef_block_simd.h b/av1/common/cdef_block_simd.h
index 358d919..b53b88b 100644
--- a/av1/common/cdef_block_simd.h
+++ b/av1/common/cdef_block_simd.h
@@ -221,6 +221,815 @@
return v128_xor(v128_add_16(sign, v128_min_s16(diff, s)), sign);
}
+#if CONFIG_CDEF_SINGLEPASS
+// sign(a - b) * min(abs(a - b), max(0, strength - (abs(a - b) >> adjdamp)))
+SIMD_INLINE v128 constrain(v256 a, v256 b, unsigned int strength,
+ unsigned int adjdamp) {
+ const v256 diff16 = v256_sub_16(a, b);
+ v128 diff = v128_pack_s16_s8(v256_high_v128(diff16), v256_low_v128(diff16));
+ const v128 sign = v128_cmplt_s8(diff, v128_zero());
+ diff = v128_abs_s8(diff);
+ return v128_xor(
+ v128_add_8(sign,
+ v128_min_u8(diff, v128_ssub_u8(v128_dup_8(strength),
+ v128_shr_u8(diff, adjdamp)))),
+ sign);
+}
+
+#if CDEF_CAP
+void SIMD_FUNC(cdef_filter_block_4x4_8)(uint8_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ UNUSED int max_unused)
+#else
+void SIMD_FUNC(cdef_filter_block_4x4_8)(uint8_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ int max)
+#endif
+{
+ v128 p0, p1, p2, p3;
+ v256 sum, row, tap, res;
+#if CDEF_CAP
+ v256 max, min, large = v256_dup_16(CDEF_VERY_LARGE);
+#endif
+ int po1 = cdef_directions[dir][0];
+ int po2 = cdef_directions[dir][1];
+#if CDEF_FULL
+ int po3 = cdef_directions[dir][2];
+#endif
+ int s1o1 = cdef_directions[(dir + 2) & 7][0];
+ int s1o2 = cdef_directions[(dir + 2) & 7][1];
+ int s2o1 = cdef_directions[(dir + 6) & 7][0];
+ int s2o2 = cdef_directions[(dir + 6) & 7][1];
+
+ const int *pri_taps = cdef_pri_taps[pri_strength & 1];
+ const int *sec_taps = cdef_sec_taps[pri_strength & 1];
+
+ if (pri_strength) pri_damping -= get_msb(pri_strength);
+ if (sec_strength) sec_damping -= get_msb(sec_strength);
+
+ sum = v256_zero();
+ row = v256_from_v64(v64_load_aligned(&in[0 * CDEF_BSTRIDE]),
+ v64_load_aligned(&in[1 * CDEF_BSTRIDE]),
+ v64_load_aligned(&in[2 * CDEF_BSTRIDE]),
+ v64_load_aligned(&in[3 * CDEF_BSTRIDE]));
+#if CDEF_CAP
+ max = min = row;
+#endif
+
+ if (pri_strength) {
+ // Primary near taps
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + po1]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + po1]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + po1]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + po1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, pri_strength, pri_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - po1]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - po1]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - po1]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - po1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[0] * (p0 + p1)
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(pri_taps[0]),
+ v256_from_v128(v128_ziphi_8(p0, p1),
+ v128_ziplo_8(p0, p1))));
+
+ // Primary far taps
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + po2]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + po2]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + po2]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + po2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, pri_strength, pri_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - po2]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - po2]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - po2]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - po2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[1] * (p0 + p1)
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(pri_taps[1]),
+ v256_from_v128(v128_ziphi_8(p0, p1),
+ v128_ziplo_8(p0, p1))));
+
+#if CDEF_FULL
+ // Primary extra taps
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + po3]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + po3]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + po3]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + po3]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, pri_strength, pri_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - po3]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - po3]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - po3]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - po3]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[2] * (p0 + p1)
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(pri_taps[2]),
+ v256_from_v128(v128_ziphi_8(p0, p1),
+ v128_ziplo_8(p0, p1))));
+#endif
+ }
+
+ if (sec_strength) {
+ // Secondary near taps
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + s1o1]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + s1o1]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + s1o1]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + s1o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, sec_strength, sec_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - s1o1]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - s1o1]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - s1o1]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - s1o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, sec_strength, sec_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + s2o1]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + s2o1]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + s2o1]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + s2o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p2 = constrain(tap, row, sec_strength, sec_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - s2o1]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - s2o1]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - s2o1]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - s2o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p3 = constrain(tap, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
+ p0 = v128_add_8(p0, p1);
+ p2 = v128_add_8(p2, p3);
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(sec_taps[0]),
+ v256_from_v128(v128_ziphi_8(p0, p2),
+ v128_ziplo_8(p0, p2))));
+
+ // Secondary far taps
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + s1o2]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + s1o2]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + s1o2]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + s1o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, sec_strength, sec_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - s1o2]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - s1o2]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - s1o2]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - s1o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, sec_strength, sec_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE + s2o2]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE + s2o2]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE + s2o2]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE + s2o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p2 = constrain(tap, row, sec_strength, sec_damping);
+ tap = v256_from_v64(v64_load_unaligned(&in[0 * CDEF_BSTRIDE - s2o2]),
+ v64_load_unaligned(&in[1 * CDEF_BSTRIDE - s2o2]),
+ v64_load_unaligned(&in[2 * CDEF_BSTRIDE - s2o2]),
+ v64_load_unaligned(&in[3 * CDEF_BSTRIDE - s2o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p3 = constrain(tap, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
+ p0 = v128_add_8(p0, p1);
+ p2 = v128_add_8(p2, p3);
+
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(sec_taps[1]),
+ v256_from_v128(v128_ziphi_8(p0, p2),
+ v128_ziplo_8(p0, p2))));
+ }
+
+ // res = row + ((sum - (sum < 0) + 8) >> 4)
+ sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
+ res = v256_add_16(sum, v256_dup_16(8));
+ res = v256_shr_n_s16(res, 4);
+ res = v256_add_16(row, res);
+#if CDEF_CAP
+ res = v256_min_s16(v256_max_s16(res, min), max);
+#else
+ res = v256_min_s16(v256_max_s16(res, v256_zero()), v256_dup_16(max));
+#endif
+ res = v256_pack_s16_u8(res, res);
+
+ p0 = v256_low_v128(res);
+ u32_store_aligned(&dst[0 * dstride], v64_high_u32(v128_high_v64(p0)));
+ u32_store_aligned(&dst[1 * dstride], v64_low_u32(v128_high_v64(p0)));
+ u32_store_aligned(&dst[2 * dstride], v64_high_u32(v128_low_v64(p0)));
+ u32_store_aligned(&dst[3 * dstride], v64_low_u32(v128_low_v64(p0)));
+}
+
+#if CDEF_CAP
+void SIMD_FUNC(cdef_filter_block_8x8_8)(uint8_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ UNUSED int max_unused)
+#else
+void SIMD_FUNC(cdef_filter_block_8x8_8)(uint8_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ int max)
+#endif
+{
+ int i;
+ v128 p0, p1, p2, p3;
+ v256 sum, row, res, tap;
+#if CDEF_CAP
+ v256 max, min, large = v256_dup_16(CDEF_VERY_LARGE);
+#endif
+ int po1 = cdef_directions[dir][0];
+ int po2 = cdef_directions[dir][1];
+#if CDEF_FULL
+ int po3 = cdef_directions[dir][2];
+#endif
+ int s1o1 = cdef_directions[(dir + 2) & 7][0];
+ int s1o2 = cdef_directions[(dir + 2) & 7][1];
+ int s2o1 = cdef_directions[(dir + 6) & 7][0];
+ int s2o2 = cdef_directions[(dir + 6) & 7][1];
+
+ const int *pri_taps = cdef_pri_taps[pri_strength & 1];
+ const int *sec_taps = cdef_sec_taps[pri_strength & 1];
+
+ if (pri_strength) pri_damping -= get_msb(pri_strength);
+ if (sec_strength) sec_damping -= get_msb(sec_strength);
+ for (i = 0; i < 8; i += 2) {
+ sum = v256_zero();
+ row = v256_from_v128(v128_load_aligned(&in[i * CDEF_BSTRIDE]),
+ v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
+
+#if CDEF_CAP
+ max = min = row;
+#endif
+ // Primary near taps
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + po1]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, pri_strength, pri_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - po1]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[0] * (p0 + p1)
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(pri_taps[0]),
+ v256_from_v128(v128_ziphi_8(p0, p1),
+ v128_ziplo_8(p0, p1))));
+
+ // Primary far taps
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + po2]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, pri_strength, pri_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - po2]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[1] * (p0 + p1)
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(pri_taps[1]),
+ v256_from_v128(v128_ziphi_8(p0, p1),
+ v128_ziplo_8(p0, p1))));
+
+#if CDEF_FULL
+ // Primary extra taps
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + po3]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po3]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, pri_strength, pri_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - po3]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po3]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[2] * (p0 + p1)
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(pri_taps[2]),
+ v256_from_v128(v128_ziphi_8(p0, p1),
+ v128_ziplo_8(p0, p1))));
+#endif
+
+ // Secondary near taps
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o1]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, sec_strength, sec_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o1]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, sec_strength, sec_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o1]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p2 = constrain(tap, row, sec_strength, sec_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o1]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p3 = constrain(tap, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
+ p0 = v128_add_8(p0, p1);
+ p2 = v128_add_8(p2, p3);
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(sec_taps[0]),
+ v256_from_v128(v128_ziphi_8(p0, p2),
+ v128_ziplo_8(p0, p2))));
+
+ // Secondary far taps
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o2]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p0 = constrain(tap, row, sec_strength, sec_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o2]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p1 = constrain(tap, row, sec_strength, sec_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o2]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p2 = constrain(tap, row, sec_strength, sec_damping);
+ tap =
+ v256_from_v128(v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o2]),
+ v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]));
+#if CDEF_CAP
+ max = v256_max_s16(max, v256_andn(tap, v256_cmpeq_16(tap, large)));
+ min = v256_min_s16(min, tap);
+#endif
+ p3 = constrain(tap, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
+ p0 = v128_add_8(p0, p1);
+ p2 = v128_add_8(p2, p3);
+ sum = v256_add_16(sum, v256_madd_us8(v256_dup_8(sec_taps[1]),
+ v256_from_v128(v128_ziphi_8(p0, p2),
+ v128_ziplo_8(p0, p2))));
+
+ // res = row + ((sum - (sum < 0) + 8) >> 4)
+ sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
+ res = v256_add_16(sum, v256_dup_16(8));
+ res = v256_shr_n_s16(res, 4);
+ res = v256_add_16(row, res);
+#if CDEF_CAP
+ res = v256_min_s16(v256_max_s16(res, min), max);
+#else
+ res = v256_min_s16(v256_max_s16(res, v256_zero()), v256_dup_16(max));
+#endif
+ res = v256_pack_s16_u8(res, res);
+
+ p0 = v256_low_v128(res);
+ v64_store_aligned(&dst[i * dstride], v128_high_v64(p0));
+ v64_store_aligned(&dst[(i + 1) * dstride], v128_low_v64(p0));
+ }
+}
+
+#if CDEF_CAP
+void SIMD_FUNC(cdef_filter_block_4x4_16)(uint16_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ UNUSED int max_unused)
+#else
+void SIMD_FUNC(cdef_filter_block_4x4_16)(uint16_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ int max)
+#endif
+{
+ int i;
+ v128 p0, p1, p2, p3, sum, row, res;
+#if CDEF_CAP
+ v128 max, min, large = v128_dup_16(CDEF_VERY_LARGE);
+#endif
+ int po1 = cdef_directions[dir][0];
+ int po2 = cdef_directions[dir][1];
+#if CDEF_FULL
+ int po3 = cdef_directions[dir][2];
+#endif
+ int s1o1 = cdef_directions[(dir + 2) & 7][0];
+ int s1o2 = cdef_directions[(dir + 2) & 7][1];
+ int s2o1 = cdef_directions[(dir + 6) & 7][0];
+ int s2o2 = cdef_directions[(dir + 6) & 7][1];
+
+ const int *pri_taps = cdef_pri_taps[pri_strength & 1];
+ const int *sec_taps = cdef_sec_taps[pri_strength & 1];
+
+ if (pri_strength) pri_damping -= get_msb(pri_strength);
+ if (sec_strength) sec_damping -= get_msb(sec_strength);
+ for (i = 0; i < 4; i += 2) {
+ sum = v128_zero();
+ row = v128_from_v64(v64_load_aligned(&in[i * CDEF_BSTRIDE]),
+ v64_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
+#if CDEF_CAP
+ min = max = row;
+#endif
+
+ // Primary near taps
+ p0 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + po1]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]));
+ p1 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - po1]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]));
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ min = v128_min_s16(v128_min_s16(min, p0), p1);
+#endif
+ p0 = constrain16(p0, row, pri_strength, pri_damping);
+ p1 = constrain16(p1, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[0] * (p0 + p1)
+ sum = v128_add_16(
+ sum, v128_mullo_s16(v128_dup_16(pri_taps[0]), v128_add_16(p0, p1)));
+
+ // Primary far taps
+ p0 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + po2]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]));
+ p1 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - po2]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]));
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ min = v128_min_s16(v128_min_s16(min, p0), p1);
+#endif
+ p0 = constrain16(p0, row, pri_strength, pri_damping);
+ p1 = constrain16(p1, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[1] * (p0 + p1)
+ sum = v128_add_16(
+ sum, v128_mullo_s16(v128_dup_16(pri_taps[1]), v128_add_16(p0, p1)));
+
+#if CDEF_FULL
+ // Primary extra taps
+ p0 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + po3]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po3]));
+ p1 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - po3]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po3]));
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ min = v128_min_s16(v128_min_s16(min, p0), p1);
+#endif
+ p0 = constrain16(p0, row, pri_strength, pri_damping);
+ p1 = constrain16(p1, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[2] * (p0 + p1)
+ sum = v128_add_16(
+ sum, v128_mullo_s16(v128_dup_16(pri_taps[2]), v128_add_16(p0, p1)));
+#endif
+
+ // Secondary near taps
+ p0 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + s1o1]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]));
+ p1 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - s1o1]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]));
+ p2 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + s2o1]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]));
+ p3 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - s2o1]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]));
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p2, v128_cmpeq_16(p2, large))),
+ v128_andn(p3, v128_cmpeq_16(p3, large)));
+ min = v128_min_s16(
+ v128_min_s16(v128_min_s16(v128_min_s16(min, p0), p1), p2), p3);
+#endif
+ p0 = constrain16(p0, row, sec_strength, sec_damping);
+ p1 = constrain16(p1, row, sec_strength, sec_damping);
+ p2 = constrain16(p2, row, sec_strength, sec_damping);
+ p3 = constrain16(p3, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
+ sum = v128_add_16(sum, v128_mullo_s16(v128_dup_16(sec_taps[0]),
+ v128_add_16(v128_add_16(p0, p1),
+ v128_add_16(p2, p3))));
+
+ // Secondary far taps
+ p0 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + s1o2]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]));
+ p1 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - s1o2]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]));
+ p2 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE + s2o2]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]));
+ p3 = v128_from_v64(v64_load_unaligned(&in[i * CDEF_BSTRIDE - s2o2]),
+ v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]));
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p2, v128_cmpeq_16(p2, large))),
+ v128_andn(p3, v128_cmpeq_16(p3, large)));
+ min = v128_min_s16(
+ v128_min_s16(v128_min_s16(v128_min_s16(min, p0), p1), p2), p3);
+#endif
+ p0 = constrain16(p0, row, sec_strength, sec_damping);
+ p1 = constrain16(p1, row, sec_strength, sec_damping);
+ p2 = constrain16(p2, row, sec_strength, sec_damping);
+ p3 = constrain16(p3, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
+ sum = v128_add_16(sum, v128_mullo_s16(v128_dup_16(sec_taps[1]),
+ v128_add_16(v128_add_16(p0, p1),
+ v128_add_16(p2, p3))));
+
+ // res = row + ((sum - (sum < 0) + 8) >> 4)
+ sum = v128_add_16(sum, v128_cmplt_s16(sum, v128_zero()));
+ res = v128_add_16(sum, v128_dup_16(8));
+ res = v128_shr_n_s16(res, 4);
+ res = v128_add_16(row, res);
+#if CDEF_CAP
+ res = v128_min_s16(v128_max_s16(res, min), max);
+#else
+ res = v128_min_s16(v128_max_s16(res, v128_zero()), v128_dup_16(max));
+#endif
+ v64_store_aligned(&dst[i * dstride], v128_high_v64(res));
+ v64_store_aligned(&dst[(i + 1) * dstride], v128_low_v64(res));
+ }
+}
+
+#if CDEF_CAP
+void SIMD_FUNC(cdef_filter_block_8x8_16)(uint16_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ UNUSED int max_unused)
+#else
+void SIMD_FUNC(cdef_filter_block_8x8_16)(uint16_t *dst, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir,
+ int pri_damping, int sec_damping,
+ int max)
+#endif
+{
+ int i;
+ v128 sum, p0, p1, p2, p3, row, res;
+#if CDEF_CAP
+ v128 max, min, large = v128_dup_16(CDEF_VERY_LARGE);
+#endif
+ int po1 = cdef_directions[dir][0];
+ int po2 = cdef_directions[dir][1];
+#if CDEF_FULL
+ int po3 = cdef_directions[dir][2];
+#endif
+ int s1o1 = cdef_directions[(dir + 2) & 7][0];
+ int s1o2 = cdef_directions[(dir + 2) & 7][1];
+ int s2o1 = cdef_directions[(dir + 6) & 7][0];
+ int s2o2 = cdef_directions[(dir + 6) & 7][1];
+
+ const int *pri_taps = cdef_pri_taps[pri_strength & 1];
+ const int *sec_taps = cdef_sec_taps[pri_strength & 1];
+
+ if (pri_strength) pri_damping -= get_msb(pri_strength);
+ if (sec_strength) sec_damping -= get_msb(sec_strength);
+
+ for (i = 0; i < 8; i++) {
+ sum = v128_zero();
+ row = v128_load_aligned(&in[i * CDEF_BSTRIDE]);
+
+#if CDEF_CAP
+ min = max = row;
+#endif
+ // Primary near taps
+ p0 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + po1]);
+ p1 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - po1]);
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ min = v128_min_s16(v128_min_s16(min, p0), p1);
+#endif
+ p0 = constrain16(p0, row, pri_strength, pri_damping);
+ p1 = constrain16(p1, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[0] * (p0 + p1)
+ sum = v128_add_16(
+ sum, v128_mullo_s16(v128_dup_16(pri_taps[0]), v128_add_16(p0, p1)));
+
+ // Primary far taps
+ p0 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + po2]);
+ p1 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - po2]);
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ min = v128_min_s16(v128_min_s16(min, p0), p1);
+#endif
+ p0 = constrain16(p0, row, pri_strength, pri_damping);
+ p1 = constrain16(p1, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[1] * (p0 + p1)
+ sum = v128_add_16(
+ sum, v128_mullo_s16(v128_dup_16(pri_taps[1]), v128_add_16(p0, p1)));
+
+#if CDEF_FULL
+ // Primary extra taps
+ p0 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + po3]);
+ p1 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - po3]);
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ min = v128_min_s16(v128_min_s16(min, p0), p1);
+#endif
+ p0 = constrain16(p0, row, pri_strength, pri_damping);
+ p1 = constrain16(p1, row, pri_strength, pri_damping);
+
+ // sum += pri_taps[2] * (p0 + p1)
+ sum = v128_add_16(
+ sum, v128_mullo_s16(v128_dup_16(pri_taps[2]), v128_add_16(p0, p1)));
+#endif
+
+ // Secondary near taps
+ p0 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o1]);
+ p1 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o1]);
+ p2 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o1]);
+ p3 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o1]);
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p2, v128_cmpeq_16(p2, large))),
+ v128_andn(p3, v128_cmpeq_16(p3, large)));
+ min = v128_min_s16(
+ v128_min_s16(v128_min_s16(v128_min_s16(min, p0), p1), p2), p3);
+#endif
+ p0 = constrain16(p0, row, sec_strength, sec_damping);
+ p1 = constrain16(p1, row, sec_strength, sec_damping);
+ p2 = constrain16(p2, row, sec_strength, sec_damping);
+ p3 = constrain16(p3, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
+ sum = v128_add_16(sum, v128_mullo_s16(v128_dup_16(sec_taps[0]),
+ v128_add_16(v128_add_16(p0, p1),
+ v128_add_16(p2, p3))));
+
+ // Secondary far taps
+ p0 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o2]);
+ p1 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o2]);
+ p2 = v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o2]);
+ p3 = v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o2]);
+#if CDEF_CAP
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p0, v128_cmpeq_16(p0, large))),
+ v128_andn(p1, v128_cmpeq_16(p1, large)));
+ max =
+ v128_max_s16(v128_max_s16(max, v128_andn(p2, v128_cmpeq_16(p2, large))),
+ v128_andn(p3, v128_cmpeq_16(p3, large)));
+ min = v128_min_s16(
+ v128_min_s16(v128_min_s16(v128_min_s16(min, p0), p1), p2), p3);
+#endif
+ p0 = constrain16(p0, row, sec_strength, sec_damping);
+ p1 = constrain16(p1, row, sec_strength, sec_damping);
+ p2 = constrain16(p2, row, sec_strength, sec_damping);
+ p3 = constrain16(p3, row, sec_strength, sec_damping);
+
+ // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
+ sum = v128_add_16(sum, v128_mullo_s16(v128_dup_16(sec_taps[1]),
+ v128_add_16(v128_add_16(p0, p1),
+ v128_add_16(p2, p3))));
+
+ // res = row + ((sum - (sum < 0) + 8) >> 4)
+ sum = v128_add_16(sum, v128_cmplt_s16(sum, v128_zero()));
+ res = v128_add_16(sum, v128_dup_16(8));
+ res = v128_shr_n_s16(res, 4);
+ res = v128_add_16(row, res);
+#if CDEF_CAP
+ res = v128_min_s16(v128_max_s16(res, min), max);
+#else
+ res = v128_min_s16(v128_max_s16(res, v128_zero()), v128_dup_16(max));
+#endif
+ v128_store_unaligned(&dst[i * dstride], res);
+ }
+}
+
+void SIMD_FUNC(cdef_filter_block)(uint8_t *dst8, uint16_t *dst16, int dstride,
+ const uint16_t *in, int pri_strength,
+ int sec_strength, int dir, int pri_damping,
+ int sec_damping, int bsize, int max) {
+ if (dst8)
+ (bsize == BLOCK_8X8 ? SIMD_FUNC(cdef_filter_block_8x8_8)
+ : SIMD_FUNC(cdef_filter_block_4x4_8))(
+ dst8, dstride, in, pri_strength, sec_strength, dir, pri_damping,
+ sec_damping, max);
+ else
+ (bsize == BLOCK_8X8 ? SIMD_FUNC(cdef_filter_block_8x8_16)
+ : SIMD_FUNC(cdef_filter_block_4x4_16))(
+ dst16, dstride, in, pri_strength, sec_strength, dir, pri_damping,
+ sec_damping, max);
+}
+
+#else
+
void SIMD_FUNC(cdef_direction_4x4)(uint16_t *y, int ystride, const uint16_t *in,
int threshold, int dir, int damping) {
int i;
@@ -364,6 +1173,7 @@
v64_store_unaligned(&dst[i * dstride], row);
}
}
+#endif
void SIMD_FUNC(copy_rect8_8bit_to_16bit)(uint16_t *dst, int dstride,
const uint8_t *src, int sstride, int v,
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 2d2f113..fb7550f 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -3011,8 +3011,12 @@
#if CONFIG_CDEF
static void setup_cdef(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
int i;
+#if CONFIG_CDEF_SINGLEPASS
+ cm->cdef_pri_damping = cm->cdef_sec_damping = aom_rb_read_literal(rb, 2) + 3;
+#else
cm->cdef_pri_damping = aom_rb_read_literal(rb, 1) + 5;
cm->cdef_sec_damping = aom_rb_read_literal(rb, 2) + 3;
+#endif
cm->cdef_bits = aom_rb_read_literal(rb, 2);
cm->nb_cdef_strengths = 1 << cm->cdef_bits;
for (i = 0; i < cm->nb_cdef_strengths; i++) {
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 4a6fbd8..c86ef3a 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -3454,8 +3454,13 @@
#if CONFIG_CDEF
static void encode_cdef(const AV1_COMMON *cm, struct aom_write_bit_buffer *wb) {
int i;
+#if CONFIG_CDEF_SINGLEPASS
+ aom_wb_write_literal(wb, cm->cdef_pri_damping - 3, 2);
+ assert(cm->cdef_pri_damping == cm->cdef_sec_damping);
+#else
aom_wb_write_literal(wb, cm->cdef_pri_damping - 5, 1);
aom_wb_write_literal(wb, cm->cdef_sec_damping - 3, 2);
+#endif
aom_wb_write_literal(wb, cm->cdef_bits, 2);
for (i = 0; i < cm->nb_cdef_strengths; i++) {
aom_wb_write_literal(wb, cm->cdef_strengths[i], CDEF_STRENGTH_BITS);
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index da2370b..f3f7799 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -4206,7 +4206,7 @@
cm->cdef_strengths[0] = 0;
cm->nb_cdef_strengths = 1;
} else {
- // Find cm->dering_level, cm->clpf_strength_u and cm->clpf_strength_v
+ // Find CDEF parameters
av1_cdef_search(cm->frame_to_show, cpi->source, cm, xd,
cpi->oxcf.speed > 0);
diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c
index 443e9e5..accc97e 100644
--- a/av1/encoder/pickcdef.c
+++ b/av1/encoder/pickcdef.c
@@ -68,11 +68,16 @@
uint64_t (**mse)[TOTAL_STRENGTHS], int sb_count,
int fast) {
uint64_t tot_mse[TOTAL_STRENGTHS][TOTAL_STRENGTHS];
+#if !CONFIG_CDEF_SINGLEPASS
const int total_strengths = fast ? REDUCED_TOTAL_STRENGTHS : TOTAL_STRENGTHS;
+#endif
int i, j;
uint64_t best_tot_mse = (uint64_t)1 << 63;
int best_id0 = 0;
int best_id1 = 0;
+#if CONFIG_CDEF_SINGLEPASS
+ const int total_strengths = fast ? REDUCED_TOTAL_STRENGTHS : TOTAL_STRENGTHS;
+#endif
memset(tot_mse, 0, sizeof(tot_mse));
for (i = 0; i < sb_count; i++) {
int gi;
@@ -305,7 +310,11 @@
int *sb_index = aom_malloc(nvfb * nhfb * sizeof(*sb_index));
int *selected_strength = aom_malloc(nvfb * nhfb * sizeof(*sb_index));
uint64_t(*mse[2])[TOTAL_STRENGTHS];
+#if CONFIG_CDEF_SINGLEPASS
+ int pri_damping = 3 + (cm->base_qindex >> 6);
+#else
int pri_damping = 6;
+#endif
int sec_damping = 3 + (cm->base_qindex >> 6);
int i;
int nb_strengths;
@@ -414,6 +423,17 @@
int xsize = (nhb << mi_wide_l2[pli]) +
CDEF_HBORDER * (fbc != nhfb - 1) + xoff;
sec_strength = gi % CDEF_SEC_STRENGTHS;
+#if CONFIG_CDEF_SINGLEPASS
+ copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE,
+ src[pli],
+ (fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff,
+ (fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff,
+ stride[pli], ysize, xsize);
+ cdef_filter_fb(NULL, tmp_dst, CDEF_BSTRIDE, in, xdec[pli], ydec[pli],
+ dir, &dirinit, var, pli, dlist, cdef_count, threshold,
+ sec_strength + (sec_strength == 3), pri_damping,
+ sec_damping, coeff_shift);
+#else
if (sec_strength == 0)
copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE,
src[pli],
@@ -425,6 +445,7 @@
pli, dlist, cdef_count, threshold,
sec_strength + (sec_strength == 3), sec_damping,
pri_damping, coeff_shift, sec_strength != 0, 1);
+#endif
curr_mse = compute_cdef_dist(
ref_coeff[pli] +
(fbr * MI_SIZE_64X64 << mi_high_l2[pli]) * stride[pli] +
diff --git a/build/cmake/aom_config_defaults.cmake b/build/cmake/aom_config_defaults.cmake
index aacb1ac..4d45a57 100644
--- a/build/cmake/aom_config_defaults.cmake
+++ b/build/cmake/aom_config_defaults.cmake
@@ -114,6 +114,7 @@
set(CONFIG_BGSPRITE 0 CACHE NUMBER "AV1 experiment flag.")
set(CONFIG_CB4X4 1 CACHE NUMBER "AV1 experiment flag.")
set(CONFIG_CDEF 1 CACHE NUMBER "AV1 experiment flag.")
+set(CONFIG_CDEF_SINGLEPASS 0 CACHE NUMBER "AV1 experiment flag.")
set(CONFIG_CFL 0 CACHE NUMBER "AV1 experiment flag.")
set(CONFIG_CHROMA_2X2 0 CACHE NUMBER "AV1 experiment flag.")
set(CONFIG_CHROMA_SUB8X8 1 CACHE NUMBER "AV1 experiment flag.")
diff --git a/configure b/configure
index ce82e73..2f2f130 100755
--- a/configure
+++ b/configure
@@ -244,6 +244,7 @@
EXPERIMENT_LIST="
fp_mb_stats
cdef
+ cdef_singlepass
var_tx
rect_tx
rect_tx_ext
@@ -561,6 +562,7 @@
enabled altref2 && enable_feature flex_refs
enabled rect_tx_ext && enable_feature rect_tx
enabled cfl && enable_feature smooth_hv
+ enabled cdef_singlepass && enable_feature cdef
if ! enabled delta_q && enabled ext_delta_q; then
log_echo "ext_delta_q requires delta_q, so disabling ext_delta_q"
diff --git a/test/cdef_test.cc b/test/cdef_test.cc
new file mode 100644
index 0000000..d2ab692
--- /dev/null
+++ b/test/cdef_test.cc
@@ -0,0 +1,419 @@
+/*
+ * Copyright (c) 2016, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+*/
+
+#include <cstdlib>
+#include <string>
+
+#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
+
+#include "./aom_config.h"
+#include "./av1_rtcd.h"
+#include "aom_ports/aom_timer.h"
+#include "av1/common/cdef_block.h"
+#include "test/acm_random.h"
+#include "test/clear_system_state.h"
+#include "test/register_state_check.h"
+#include "test/util.h"
+
+using libaom_test::ACMRandom;
+
+namespace {
+
+typedef std::tr1::tuple<cdef_filter_block_func, cdef_filter_block_func, int>
+ cdef_dir_param_t;
+
+class CDEFBlockTest : public ::testing::TestWithParam<cdef_dir_param_t> {
+ public:
+ virtual ~CDEFBlockTest() {}
+ virtual void SetUp() {
+ cdef = GET_PARAM(0);
+ ref_cdef = GET_PARAM(1);
+ bsize = GET_PARAM(2);
+ }
+
+ virtual void TearDown() { libaom_test::ClearSystemState(); }
+
+ protected:
+ int bsize;
+ cdef_filter_block_func cdef;
+ cdef_filter_block_func ref_cdef;
+};
+
+typedef CDEFBlockTest CDEFSpeedTest;
+
+void test_cdef(int bsize, int iterations, cdef_filter_block_func cdef,
+ cdef_filter_block_func ref_cdef) {
+ const int size = 8;
+ const int ysize = size + 2 * CDEF_VBORDER;
+ ACMRandom rnd(ACMRandom::DeterministicSeed());
+ DECLARE_ALIGNED(16, uint16_t, s[ysize * CDEF_BSTRIDE]);
+ DECLARE_ALIGNED(16, static uint16_t, d[size * size]);
+ DECLARE_ALIGNED(16, static uint16_t, ref_d[size * size]);
+ memset(ref_d, 0, sizeof(ref_d));
+ memset(d, 0, sizeof(d));
+
+ int error = 0, pristrength = 0, secstrength, dir;
+ int boundary, pridamping, secdamping, depth, bits, level, count,
+ errdepth = 0, errpristrength = 0, errsecstrength = 0, errboundary = 0,
+ errpridamping = 0, errsecdamping = 0;
+ unsigned int pos = 0;
+
+ for (boundary = 0; boundary < 16; boundary++) {
+ for (depth = 8; depth <= 12; depth += 2) {
+ for (pridamping = 3 + depth - 8;
+ pridamping < 7 - 3 * !!boundary + depth - 8; pridamping++) {
+ for (secdamping = 3 + depth - 8;
+ secdamping < 7 - 3 * !!boundary + depth - 8; secdamping++) {
+ for (count = 0; count < iterations; count++) {
+ for (level = 0; level < (1 << depth) && !error;
+ level += (2 + 6 * !!boundary) << (depth - 8)) {
+ for (bits = 1; bits <= depth && !error;
+ bits += 1 + 3 * !!boundary) {
+ for (unsigned int i = 0; i < sizeof(s) / sizeof(*s); i++)
+ s[i] = clamp((rnd.Rand16() & ((1 << bits) - 1)) + level, 0,
+ (1 << depth) - 1);
+ if (boundary) {
+ if (boundary & 1) { // Left
+ for (int i = 0; i < ysize; i++)
+ for (int j = 0; j < CDEF_HBORDER; j++)
+ s[i * CDEF_BSTRIDE + j] = CDEF_VERY_LARGE;
+ }
+ if (boundary & 2) { // Right
+ for (int i = 0; i < ysize; i++)
+ for (int j = CDEF_HBORDER + size; j < CDEF_BSTRIDE; j++)
+ s[i * CDEF_BSTRIDE + j] = CDEF_VERY_LARGE;
+ }
+ if (boundary & 4) { // Above
+ for (int i = 0; i < CDEF_VBORDER; i++)
+ for (int j = 0; j < CDEF_BSTRIDE; j++)
+ s[i * CDEF_BSTRIDE + j] = CDEF_VERY_LARGE;
+ }
+ if (boundary & 8) { // Below
+ for (int i = CDEF_VBORDER + size; i < ysize; i++)
+ for (int j = 0; j < CDEF_BSTRIDE; j++)
+ s[i * CDEF_BSTRIDE + j] = CDEF_VERY_LARGE;
+ }
+ }
+ for (dir = 0; dir < 8; dir++) {
+ for (pristrength = 0;
+ pristrength <= 19 << (depth - 8) && !error;
+ pristrength += (1 + 4 * !!boundary) << (depth - 8)) {
+ if (pristrength == 16) pristrength = 19;
+ for (secstrength = 0;
+ secstrength <= 4 << (depth - 8) && !error;
+ secstrength += 1 << (depth - 8)) {
+ if (secstrength == 3 << (depth - 8)) continue;
+ ref_cdef(depth == 8 ? (uint8_t *)ref_d : 0, ref_d, size,
+ s + CDEF_HBORDER + CDEF_VBORDER * CDEF_BSTRIDE,
+ pristrength, secstrength, dir, pridamping,
+ secdamping, bsize, (1 << depth) - 1);
+ // If cdef and ref_cdef are the same, we're just testing
+ // speed
+ if (cdef != ref_cdef)
+ ASM_REGISTER_STATE_CHECK(
+ cdef(depth == 8 ? (uint8_t *)d : 0, d, size,
+ s + CDEF_HBORDER + CDEF_VBORDER * CDEF_BSTRIDE,
+ pristrength, secstrength, dir, pridamping,
+ secdamping, bsize, (1 << depth) - 1));
+ if (ref_cdef != cdef) {
+ for (pos = 0;
+ pos<sizeof(d) / sizeof(*d)>> (depth == 8) &&
+ !error;
+ pos++) {
+ error = ref_d[pos] != d[pos];
+ errdepth = depth;
+ errpristrength = pristrength;
+ errsecstrength = secstrength;
+ errboundary = boundary;
+ errpridamping = pridamping;
+ errsecdamping = secdamping;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ pos--;
+ EXPECT_EQ(0, error) << "Error: CDEFBlockTest, SIMD and C mismatch."
+ << std::endl
+ << "First error at " << pos % size << "," << pos / size
+ << " (" << (int16_t)ref_d[pos] << " : " << (int16_t)d[pos]
+ << ") " << std::endl
+ << "pristrength: " << errpristrength << std::endl
+ << "pridamping: " << errpridamping << std::endl
+ << "secstrength: " << errsecstrength << std::endl
+ << "secdamping: " << errsecdamping << std::endl
+ << "depth: " << errdepth << std::endl
+ << "size: " << bsize << std::endl
+ << "boundary: " << errboundary << std::endl
+ << std::endl;
+}
+
+void test_cdef_speed(int bsize, int iterations, cdef_filter_block_func cdef,
+ cdef_filter_block_func ref_cdef) {
+ aom_usec_timer ref_timer;
+ aom_usec_timer timer;
+
+ aom_usec_timer_start(&ref_timer);
+ test_cdef(bsize, iterations, ref_cdef, ref_cdef);
+ aom_usec_timer_mark(&ref_timer);
+ int ref_elapsed_time = (int)aom_usec_timer_elapsed(&ref_timer);
+
+ aom_usec_timer_start(&timer);
+ test_cdef(bsize, iterations, cdef, cdef);
+ aom_usec_timer_mark(&timer);
+ int elapsed_time = (int)aom_usec_timer_elapsed(&timer);
+
+#if 0
+ std::cout << "[ ] C time = " << ref_elapsed_time / 1000
+ << " ms, SIMD time = " << elapsed_time / 1000 << " ms" << std::endl;
+#endif
+
+ EXPECT_GT(ref_elapsed_time, elapsed_time)
+ << "Error: CDEFSpeedTest, SIMD slower than C." << std::endl
+ << "C time: " << ref_elapsed_time << " us" << std::endl
+ << "SIMD time: " << elapsed_time << " us" << std::endl;
+}
+
+typedef int (*find_dir_t)(const uint16_t *img, int stride, int32_t *var,
+ int coeff_shift);
+
+typedef std::tr1::tuple<find_dir_t, find_dir_t> find_dir_param_t;
+
+class CDEFFindDirTest : public ::testing::TestWithParam<find_dir_param_t> {
+ public:
+ virtual ~CDEFFindDirTest() {}
+ virtual void SetUp() {
+ finddir = GET_PARAM(0);
+ ref_finddir = GET_PARAM(1);
+ }
+
+ virtual void TearDown() { libaom_test::ClearSystemState(); }
+
+ protected:
+ find_dir_t finddir;
+ find_dir_t ref_finddir;
+};
+
+typedef CDEFFindDirTest CDEFFindDirSpeedTest;
+
+void test_finddir(int (*finddir)(const uint16_t *img, int stride, int32_t *var,
+ int coeff_shift),
+ int (*ref_finddir)(const uint16_t *img, int stride,
+ int32_t *var, int coeff_shift)) {
+ const int size = 8;
+ ACMRandom rnd(ACMRandom::DeterministicSeed());
+ DECLARE_ALIGNED(16, uint16_t, s[size * size]);
+
+ int error = 0;
+ int depth, bits, level, count, errdepth = 0;
+ int ref_res = 0, res = 0;
+ int32_t ref_var = 0, var = 0;
+
+ for (depth = 8; depth <= 12 && !error; depth += 2) {
+ for (count = 0; count < 512 && !error; count++) {
+ for (level = 0; level < (1 << depth) && !error;
+ level += 1 << (depth - 8)) {
+ for (bits = 1; bits <= depth && !error; bits++) {
+ for (unsigned int i = 0; i < sizeof(s) / sizeof(*s); i++)
+ s[i] = clamp((rnd.Rand16() & ((1 << bits) - 1)) + level, 0,
+ (1 << depth) - 1);
+ for (int c = 0; c < 1 + 9 * (finddir == ref_finddir); c++)
+ ref_res = ref_finddir(s, size, &ref_var, depth - 8);
+ if (finddir != ref_finddir)
+ ASM_REGISTER_STATE_CHECK(res = finddir(s, size, &var, depth - 8));
+ if (ref_finddir != finddir) {
+ if (res != ref_res || var != ref_var) error = 1;
+ errdepth = depth;
+ }
+ }
+ }
+ }
+ }
+
+ EXPECT_EQ(0, error) << "Error: CDEFFindDirTest, SIMD and C mismatch."
+ << std::endl
+ << "return: " << res << " : " << ref_res << std::endl
+ << "var: " << var << " : " << ref_var << std::endl
+ << "depth: " << errdepth << std::endl
+ << std::endl;
+}
+
+void test_finddir_speed(int (*finddir)(const uint16_t *img, int stride,
+ int32_t *var, int coeff_shift),
+ int (*ref_finddir)(const uint16_t *img, int stride,
+ int32_t *var, int coeff_shift)) {
+ aom_usec_timer ref_timer;
+ aom_usec_timer timer;
+
+ aom_usec_timer_start(&ref_timer);
+ test_finddir(ref_finddir, ref_finddir);
+ aom_usec_timer_mark(&ref_timer);
+ int ref_elapsed_time = (int)aom_usec_timer_elapsed(&ref_timer);
+
+ aom_usec_timer_start(&timer);
+ test_finddir(finddir, finddir);
+ aom_usec_timer_mark(&timer);
+ int elapsed_time = (int)aom_usec_timer_elapsed(&timer);
+
+#if 0
+ std::cout << "[ ] C time = " << ref_elapsed_time / 1000
+ << " ms, SIMD time = " << elapsed_time / 1000 << " ms" << std::endl;
+#endif
+
+ EXPECT_GT(ref_elapsed_time, elapsed_time)
+ << "Error: CDEFFindDirSpeedTest, SIMD slower than C." << std::endl
+ << "C time: " << ref_elapsed_time << " us" << std::endl
+ << "SIMD time: " << elapsed_time << " us" << std::endl;
+}
+
+TEST_P(CDEFBlockTest, TestSIMDNoMismatch) {
+ test_cdef(bsize, 1, cdef, ref_cdef);
+}
+
+TEST_P(CDEFSpeedTest, DISABLED_TestSpeed) {
+ test_cdef_speed(bsize, 4, cdef, ref_cdef);
+}
+
+TEST_P(CDEFFindDirTest, TestSIMDNoMismatch) {
+ test_finddir(finddir, ref_finddir);
+}
+
+TEST_P(CDEFFindDirSpeedTest, DISABLED_TestSpeed) {
+ test_finddir_speed(finddir, ref_finddir);
+}
+
+using std::tr1::make_tuple;
+
+// VS compiling for 32 bit targets does not support vector types in
+// structs as arguments, which makes the v256 type of the intrinsics
+// hard to support, so optimizations for this target are disabled.
+#if defined(_WIN64) || !defined(_MSC_VER) || defined(__clang__)
+#if HAVE_SSE2
+INSTANTIATE_TEST_CASE_P(
+ SSE2, CDEFBlockTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_sse2, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_sse2, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(SSE2, CDEFFindDirTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_sse2,
+ &cdef_find_dir_c)));
+#endif
+#if HAVE_SSSE3
+INSTANTIATE_TEST_CASE_P(
+ SSSE3, CDEFBlockTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_ssse3, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_ssse3, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(SSSE3, CDEFFindDirTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_ssse3,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_CASE_P(
+ SSE4_1, CDEFBlockTest,
+ ::testing::Values(make_tuple(&cdef_filter_block_sse4_1,
+ &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_sse4_1,
+ &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(SSE4_1, CDEFFindDirTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_sse4_1,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_AVX2
+INSTANTIATE_TEST_CASE_P(
+ AVX2, CDEFBlockTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_avx2, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_avx2, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(AVX2, CDEFFindDirTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_avx2,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_CASE_P(
+ NEON, CDEFBlockTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_neon, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_neon, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(NEON, CDEFFindDirTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_neon,
+ &cdef_find_dir_c)));
+#endif
+
+// Test speed for all supported architectures
+#if HAVE_SSE2
+INSTANTIATE_TEST_CASE_P(
+ SSE2, CDEFSpeedTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_sse2, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_sse2, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(SSE2, CDEFFindDirSpeedTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_sse2,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_SSSE3
+INSTANTIATE_TEST_CASE_P(
+ SSSE3, CDEFSpeedTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_ssse3, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_ssse3, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(SSSE3, CDEFFindDirSpeedTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_ssse3,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_SSE4_1
+INSTANTIATE_TEST_CASE_P(
+ SSE4_1, CDEFSpeedTest,
+ ::testing::Values(make_tuple(&cdef_filter_block_sse4_1,
+ &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_sse4_1,
+ &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(SSE4_1, CDEFFindDirSpeedTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_sse4_1,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_AVX2
+INSTANTIATE_TEST_CASE_P(
+ AVX2, CDEFSpeedTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_avx2, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_avx2, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(AVX2, CDEFFindDirSpeedTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_avx2,
+ &cdef_find_dir_c)));
+#endif
+
+#if HAVE_NEON
+INSTANTIATE_TEST_CASE_P(
+ NEON, CDEFSpeedTest,
+ ::testing::Values(
+ make_tuple(&cdef_filter_block_neon, &cdef_filter_block_c, BLOCK_4X4),
+ make_tuple(&cdef_filter_block_neon, &cdef_filter_block_c, BLOCK_8X8)));
+INSTANTIATE_TEST_CASE_P(NEON, CDEFFindDirSpeedTest,
+ ::testing::Values(make_tuple(&cdef_find_dir_neon,
+ &cdef_find_dir_c)));
+#endif
+
+#endif // defined(_WIN64) || !defined(_MSC_VER)
+} // namespace
diff --git a/test/test.cmake b/test/test.cmake
index 43b975c..b035184 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -137,10 +137,16 @@
"${AOM_ROOT}/test/simd_cmp_impl.h")
if (CONFIG_CDEF)
- set(AOM_UNIT_TEST_COMMON_SOURCES
- ${AOM_UNIT_TEST_COMMON_SOURCES}
- "${AOM_ROOT}/test/clpf_test.cc"
- "${AOM_ROOT}/test/dering_test.cc")
+ if (CONFIG_CDEF_SINGLEPASS)
+ set(AOM_UNIT_TEST_COMMON_SOURCES
+ ${AOM_UNIT_TEST_COMMON_SOURCES}
+ "${AOM_ROOT}/test/cdef_test.cc")
+ else ()
+ set(AOM_UNIT_TEST_COMMON_SOURCES
+ ${AOM_UNIT_TEST_COMMON_SOURCES}
+ "${AOM_ROOT}/test/clpf_test.cc"
+ "${AOM_ROOT}/test/dering_test.cc")
+ endif ()
endif ()
# Omit 4-tap filter intra predictor test-- currently a 3-tap filter is in
diff --git a/test/test.mk b/test/test.mk
index 2f99ab7..9f60d57 100644
--- a/test/test.mk
+++ b/test/test.mk
@@ -135,8 +135,12 @@
LIBAOM_TEST_SRCS-$(CONFIG_ADAPT_SCAN) += scan_test.cc
LIBAOM_TEST_SRCS-yes += convolve_test.cc
LIBAOM_TEST_SRCS-yes += lpf_8_test.cc
+ifeq ($(CONFIG_CDEF_SINGLEPASS),yes)
+LIBAOM_TEST_SRCS-$(CONFIG_CDEF) += cdef_test.cc
+else
LIBAOM_TEST_SRCS-$(CONFIG_CDEF) += dering_test.cc
LIBAOM_TEST_SRCS-$(CONFIG_CDEF) += clpf_test.cc
+endif
LIBAOM_TEST_SRCS-yes += simd_cmp_impl.h
LIBAOM_TEST_SRCS-$(HAVE_SSE2) += simd_cmp_sse2.cc
LIBAOM_TEST_SRCS-$(HAVE_SSSE3) += simd_cmp_ssse3.cc
