[CFL] Constant Propagated TX_SIZE for subsampling Macros to generate subsampling functions for each TX_SIZE supported by CfL. This allows the compiler to inline a generic version of the function into each variation. The benefit of this is that the width and the height are constant inside this function. The subsampling type is also constant propagated, but only 4:2:0 is currently implemented. Change-Id: I16cb33d1a7bfd1ec57cd9f68cb17116068a8e76a
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index 5f848f4..a05b1f7 100755 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -39,10 +39,10 @@ #if CONFIG_CFL /* Function pointers return by CfL functions */ typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride, - int16_t *output_q3, int width, int height); + int16_t *output_q3); typedef void (*cfl_subsample_hbd_fn)(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height); + int16_t *output_q3); typedef void (*cfl_subtract_average_fn)(int16_t *pred_buf_q3); @@ -569,11 +569,8 @@ add_proto qw/cfl_subtract_average_fn get_subtract_average_fn/, "TX_SIZE tx_size"; specialize qw/get_subtract_average_fn sse2 avx2 neon/; - add_proto qw/cfl_subsample_lbd_fn get_subsample_lbd_fn/, "int sub_x, int sub_y"; - specialize qw/get_subsample_lbd_fn ssse3 avx2/; - - add_proto qw/cfl_subsample_hbd_fn get_subsample_hbd_fn/, "int sub_x, int sub_y"; - specialize qw/get_subsample_hbd_fn ssse3 avx2/; + add_proto qw/cfl_subsample_lbd_fn cfl_get_luma_subsampling_420_lbd/, "TX_SIZE tx_size"; + specialize qw/cfl_get_luma_subsampling_420_lbd ssse3 avx2/; add_proto qw/cfl_predict_lbd_fn get_predict_lbd_fn/, "TX_SIZE tx_size"; specialize qw/get_predict_lbd_fn ssse3 avx2/;
diff --git a/av1/common/cfl.c b/av1/common/cfl.c index 8643f1d..46cc8d0 100644 --- a/av1/common/cfl.c +++ b/av1/common/cfl.c
@@ -227,6 +227,24 @@ alpha_q3); } +// Null function used for invalid tx_sizes +void cfl_subsample_lbd_null(const uint8_t *input, int input_stride, + int16_t *output_q3) { + (void)input; + (void)input_stride; + (void)output_q3; + assert(0); +} + +// Null function used for invalid tx_sizes +void cfl_subsample_hbd_null(const uint16_t *input, int input_stride, + int16_t *output_q3) { + (void)input; + (void)input_stride; + (void)output_q3; + assert(0); +} + static void cfl_luma_subsampling_420_lbd_c(const uint8_t *input, int input_stride, int16_t *output_q3, int width, int height) { @@ -241,8 +259,9 @@ } } -void cfl_luma_subsampling_422_lbd_c(const uint8_t *input, int input_stride, - int16_t *output_q3, int width, int height) { +static void cfl_luma_subsampling_422_lbd_c(const uint8_t *input, + int input_stride, int16_t *output_q3, + int width, int height) { assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE); for (int j = 0; j < height; j++) { for (int i = 0; i < width; i += 2) { @@ -253,8 +272,9 @@ } } -void cfl_luma_subsampling_444_lbd_c(const uint8_t *input, int input_stride, - int16_t *output_q3, int width, int height) { +static void cfl_luma_subsampling_444_lbd_c(const uint8_t *input, + int input_stride, int16_t *output_q3, + int width, int height) { assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE); for (int j = 0; j < height; j++) { for (int i = 0; i < width; i++) { @@ -265,8 +285,9 @@ } } -void cfl_luma_subsampling_420_hbd_c(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height) { +static void cfl_luma_subsampling_420_hbd_c(const uint16_t *input, + int input_stride, int16_t *output_q3, + int width, int height) { for (int j = 0; j < height; j += 2) { for (int i = 0; i < width; i += 2) { const int bot = i + input_stride; @@ -278,8 +299,9 @@ } } -void cfl_luma_subsampling_422_hbd_c(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height) { +static void cfl_luma_subsampling_422_hbd_c(const uint16_t *input, + int input_stride, int16_t *output_q3, + int width, int height) { assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE); for (int j = 0; j < height; j++) { for (int i = 0; i < width; i += 2) { @@ -290,8 +312,9 @@ } } -void cfl_luma_subsampling_444_hbd_c(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height) { +static void cfl_luma_subsampling_444_hbd_c(const uint16_t *input, + int input_stride, int16_t *output_q3, + int width, int height) { assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE); for (int j = 0; j < height; j++) { for (int i = 0; i < width; i++) { @@ -302,8 +325,90 @@ } } +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 422 SIMD +// will be implemented +CFL_SUBSAMPLE_FUNCTIONS(c, 422, lbd) + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +CFL_SUBSAMPLE_FUNCTIONS(c, 444, lbd) + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +CFL_SUBSAMPLE_FUNCTIONS(c, 420, hbd) + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +CFL_SUBSAMPLE_FUNCTIONS(c, 422, hbd) + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +CFL_SUBSAMPLE_FUNCTIONS(c, 444, hbd) + CFL_GET_SUBSAMPLE_FUNCTION(c) +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +cfl_subsample_hbd_fn cfl_get_luma_subsampling_420_hbd_c(TX_SIZE tx_size) { + CFL_SUBSAMPLE_FUNCTION_ARRAY(c, 420, hbd) + return subfn_420[tx_size]; +} + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +cfl_subsample_hbd_fn cfl_get_luma_subsampling_422_hbd_c(TX_SIZE tx_size) { + CFL_SUBSAMPLE_FUNCTION_ARRAY(c, 422, hbd) + return subfn_422[tx_size]; +} + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +cfl_subsample_hbd_fn cfl_get_luma_subsampling_444_hbd_c(TX_SIZE tx_size) { + CFL_SUBSAMPLE_FUNCTION_ARRAY(c, 444, hbd) + return subfn_444[tx_size]; +} + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 422 SIMD +// will be implemented +cfl_subsample_lbd_fn cfl_get_luma_subsampling_422_lbd_c(TX_SIZE tx_size) { + CFL_SUBSAMPLE_FUNCTION_ARRAY(c, 422, lbd) + return subfn_422[tx_size]; +} + +// TODO(ltrudeau) Move into the CFL_GET_SUBSAMPLE_FUNCTION when LBD 444 SIMD +// will be implemented +cfl_subsample_lbd_fn cfl_get_luma_subsampling_444_lbd_c(TX_SIZE tx_size) { + CFL_SUBSAMPLE_FUNCTION_ARRAY(c, 444, lbd) + return subfn_444[tx_size]; +} + +static inline cfl_subsample_hbd_fn cfl_subsampling_hbd(TX_SIZE tx_size, + int sub_x, int sub_y) { + if (sub_x == 1) { + if (sub_y == 1) { + // TODO(ltrudeau) Remove _c when HBD 420 SIMD is added + return cfl_get_luma_subsampling_420_hbd_c(tx_size); + } + // TODO(ltrudeau) Remove _c when HBD 422 SIMD is added + return cfl_get_luma_subsampling_422_hbd_c(tx_size); + } + // TODO(ltrudeau) Remove _c when HBD 444 SIMD is added + return cfl_get_luma_subsampling_444_hbd_c(tx_size); +} + +static inline cfl_subsample_lbd_fn cfl_subsampling_lbd(TX_SIZE tx_size, + int sub_x, int sub_y) { + if (sub_x == 1) { + if (sub_y == 1) { + return cfl_get_luma_subsampling_420_lbd(tx_size); + } + // TODO(ltrudeau) Remove _c when LBD 422 SIMD is added + return cfl_get_luma_subsampling_422_lbd_c(tx_size); + } + // TODO(ltrudeau) Remove _c when LBD 444 SIMD is added + return cfl_get_luma_subsampling_444_lbd_c(tx_size); +} + static void cfl_store(CFL_CTX *cfl, const uint8_t *input, int input_stride, int row, int col, TX_SIZE tx_size, int use_hbd) { const int width = tx_size_wide[tx_size]; @@ -339,13 +444,12 @@ cfl->pred_buf_q3 + (store_row * CFL_BUF_LINE + store_col); if (use_hbd) { - const uint16_t *input_16 = CONVERT_TO_SHORTPTR(input); - get_subsample_hbd_fn(sub_x, sub_y)(input_16, input_stride, pred_buf_q3, - width, height); - return; + cfl_subsampling_hbd(tx_size, sub_x, sub_y)(CONVERT_TO_SHORTPTR(input), + input_stride, pred_buf_q3); + } else { + cfl_subsampling_lbd(tx_size, sub_x, sub_y)(input, input_stride, + pred_buf_q3); } - get_subsample_lbd_fn(sub_x, sub_y)(input, input_stride, pred_buf_q3, width, - height); } // Adjust the row and column of blocks smaller than 8X8, as chroma-referenced
diff --git a/av1/common/cfl.h b/av1/common/cfl.h index b925972..3c1494b 100644 --- a/av1/common/cfl.h +++ b/av1/common/cfl.h
@@ -45,37 +45,80 @@ void cfl_load_dc_pred(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride, TX_SIZE tx_size, CFL_PRED_TYPE pred_plane); -// TODO(ltrudeau) Remove this when HBD 420 SIMD is added -void cfl_luma_subsampling_420_hbd_c(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height); +// Null function used for invalid tx_sizes +void cfl_subsample_lbd_null(const uint8_t *input, int input_stride, + int16_t *output_q3); -// TODO(ltrudeau) Remove this when HBD 422 SIMD is added -void cfl_luma_subsampling_422_hbd_c(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height); +// Null function used for invalid tx_sizes +void cfl_subsample_hbd_null(const uint16_t *input, int input_stride, + int16_t *output_q3); -// TODO(ltrudeau) Remove this when HBD 444 SIMD is added -void cfl_luma_subsampling_444_hbd_c(const uint16_t *input, int input_stride, - int16_t *output_q3, int width, int height); +// Allows the CFL_SUBSAMPLE function to switch types depending on the bitdepth. +#define CFL_SUBSAMPLE_INPUT_TYPE_lbd_ const uint8_t *input +#define CFL_SUBSAMPLE_INPUT_TYPE_hbd_ const uint16_t *input -// TODO(ltrudeau) Remove this when LBD 422 SIMD is added -void cfl_luma_subsampling_422_lbd_c(const uint8_t *input, int input_stride, - int16_t *output_q3, int width, int height); -// TODO(ltrudeau) Remove this when LBD 444 SIMD is added -void cfl_luma_subsampling_444_lbd_c(const uint8_t *input, int input_stride, - int16_t *output_q3, int width, int height); +// Declare a size-specific wrapper for the size-generic function. The compiler +// will inline the size generic function in here, the advantage is that the size +// will be constant allowing for loop unrolling and other constant propagated +// goodness. +#define CFL_SUBSAMPLE(arch, sub, bd, width, height) \ + void subsample_##bd##_##sub##_##width##x##height##_##arch( \ + CFL_SUBSAMPLE_INPUT_TYPE_##bd##_, int input_stride, \ + int16_t *output_q3) { \ + cfl_luma_subsampling_##sub##_##bd##_##arch(input, input_stride, output_q3, \ + width, height); \ + } -#define CFL_GET_SUBSAMPLE_FUNCTION(arch) \ - cfl_subsample_lbd_fn get_subsample_lbd_fn_##arch(int sub_x, int sub_y) { \ - if (sub_x == 1) \ - return (sub_y == 1) ? cfl_luma_subsampling_420_lbd_##arch \ - : cfl_luma_subsampling_422_lbd_c; \ - return cfl_luma_subsampling_444_lbd_c; \ - } \ - cfl_subsample_hbd_fn get_subsample_hbd_fn_##arch(int sub_x, int sub_y) { \ - if (sub_x == 1) \ - return (sub_y == 1) ? cfl_luma_subsampling_420_hbd_c \ - : cfl_luma_subsampling_422_hbd_c; \ - return cfl_luma_subsampling_444_hbd_c; \ +// Declare size-specific wrappers for all valid CfL sizes. +#define CFL_SUBSAMPLE_FUNCTIONS(arch, sub, bd) \ + CFL_SUBSAMPLE(arch, sub, bd, 4, 4) \ + CFL_SUBSAMPLE(arch, sub, bd, 8, 8) \ + CFL_SUBSAMPLE(arch, sub, bd, 16, 16) \ + CFL_SUBSAMPLE(arch, sub, bd, 32, 32) \ + CFL_SUBSAMPLE(arch, sub, bd, 4, 8) \ + CFL_SUBSAMPLE(arch, sub, bd, 8, 4) \ + CFL_SUBSAMPLE(arch, sub, bd, 8, 16) \ + CFL_SUBSAMPLE(arch, sub, bd, 16, 8) \ + CFL_SUBSAMPLE(arch, sub, bd, 16, 32) \ + CFL_SUBSAMPLE(arch, sub, bd, 32, 16) \ + CFL_SUBSAMPLE(arch, sub, bd, 4, 16) \ + CFL_SUBSAMPLE(arch, sub, bd, 16, 4) \ + CFL_SUBSAMPLE(arch, sub, bd, 8, 32) \ + CFL_SUBSAMPLE(arch, sub, bd, 32, 8) + +// Declare an architecture-specific array of function pointers for size-specific +// wrappers. +#define CFL_SUBSAMPLE_FUNCTION_ARRAY(arch, sub, bd) \ + static const cfl_subsample_##bd##_fn subfn_##sub[TX_SIZES_ALL] = { \ + subsample_##bd##_##sub##_4x4_##arch, /* 4x4 */ \ + subsample_##bd##_##sub##_8x8_##arch, /* 8x8 */ \ + subsample_##bd##_##sub##_16x16_##arch, /* 16x16 */ \ + subsample_##bd##_##sub##_32x32_##arch, /* 32x32 */ \ + cfl_subsample_##bd##_null, /* 64x64 (invalid CFL size) */ \ + subsample_##bd##_##sub##_4x8_##arch, /* 4x8 */ \ + subsample_##bd##_##sub##_8x4_##arch, /* 8x4 */ \ + subsample_##bd##_##sub##_8x16_##arch, /* 8x16 */ \ + subsample_##bd##_##sub##_16x8_##arch, /* 16x8 */ \ + subsample_##bd##_##sub##_16x32_##arch, /* 16x32 */ \ + subsample_##bd##_##sub##_32x16_##arch, /* 32x16 */ \ + cfl_subsample_##bd##_null, /* 32x64 (invalid CFL size) */ \ + cfl_subsample_##bd##_null, /* 64x32 (invalid CFL size) */ \ + subsample_##bd##_##sub##_4x16_##arch, /* 4x16 */ \ + subsample_##bd##_##sub##_16x4_##arch, /* 16x4 */ \ + subsample_##bd##_##sub##_8x32_##arch, /* 8x32 */ \ + subsample_##bd##_##sub##_32x8_##arch, /* 32x8 */ \ + cfl_subsample_##bd##_null, /* 16x64 (invalid CFL size) */ \ + cfl_subsample_##bd##_null, /* 64x16 (invalid CFL size) */ \ + }; + +// The RTCD script does not support passing in the an array, so we wrap it in +// this function. +#define CFL_GET_SUBSAMPLE_FUNCTION(arch) \ + CFL_SUBSAMPLE_FUNCTIONS(arch, 420, lbd) \ + cfl_subsample_lbd_fn cfl_get_luma_subsampling_420_lbd_##arch( \ + TX_SIZE tx_size) { \ + CFL_SUBSAMPLE_FUNCTION_ARRAY(arch, 420, lbd) \ + return subfn_420[tx_size]; \ } // Null function used for invalid tx_sizes
diff --git a/test/cfl_test.cc b/test/cfl_test.cc index 7ef62cc..98f4ec2 100644 --- a/test/cfl_test.cc +++ b/test/cfl_test.cc
@@ -32,7 +32,7 @@ make_tuple(TX_32X16, &function), make_tuple(TX_32X32, &function) namespace { -typedef cfl_subsample_lbd_fn (*get_subsample_fn)(int sub_x, int sub_y); +typedef cfl_subsample_lbd_fn (*get_subsample_fn)(TX_SIZE tx_size); typedef cfl_predict_lbd_fn (*get_predict_fn)(TX_SIZE tx_size); @@ -212,9 +212,9 @@ for (int i = 0; i < width; i++) { chroma_pels[j * CFL_BUF_LINE + i] = dc; chroma_pels_ref[j * CFL_BUF_LINE + i] = dc; - sub_luma_pels_ref[j * CFL_BUF_LINE + i] = - sub_luma_pels[j * CFL_BUF_LINE + i] = - rnd(1 << bd) - (1 << (bd - 1)); + int rand_val = rnd(1 << bd) - (1 << (bd - 1)); + sub_luma_pels_ref[j * CFL_BUF_LINE + i] = rand_val; + sub_luma_pels[j * CFL_BUF_LINE + i] = rand_val; } } } @@ -271,14 +271,15 @@ TEST_P(CFLSubsampleTest, SubsampleTest) { const int width = Width(); const int height = Height(); + const TX_SIZE tx_size = Tx_size(); const int sub_width = width >> 1; const int sub_height = height >> 1; for (int it = 0; it < NUM_ITERATIONS; it++) { init(width, height); - subsample(1, 1)(luma_pels, CFL_BUF_LINE, sub_luma_pels, width, height); - get_subsample_lbd_fn_c(1, 1)(luma_pels_ref, CFL_BUF_LINE, sub_luma_pels_ref, - width, height); + subsample(tx_size)(luma_pels, CFL_BUF_LINE, sub_luma_pels); + cfl_get_luma_subsampling_420_lbd_c(tx_size)(luma_pels_ref, CFL_BUF_LINE, + sub_luma_pels_ref); for (int j = 0; j < sub_height; j++) { for (int i = 0; i < sub_width; i++) { ASSERT_EQ(sub_luma_pels_ref[j * CFL_BUF_LINE + i], @@ -291,6 +292,7 @@ TEST_P(CFLSubsampleTest, DISABLED_SubsampleSpeedTest) { const int width = Width(); const int height = Height(); + const TX_SIZE tx_size = Tx_size(); aom_usec_timer ref_timer; aom_usec_timer timer; @@ -298,16 +300,15 @@ init(width, height); aom_usec_timer_start(&ref_timer); for (int k = 0; k < NUM_ITERATIONS_SPEED; k++) { - get_subsample_lbd_fn_c(1, 1)(luma_pels, CFL_BUF_LINE, sub_luma_pels, width, - height); + cfl_get_luma_subsampling_420_lbd_c(tx_size)(luma_pels, CFL_BUF_LINE, + sub_luma_pels); } aom_usec_timer_mark(&ref_timer); int ref_elapsed_time = (int)aom_usec_timer_elapsed(&ref_timer); aom_usec_timer_start(&timer); for (int k = 0; k < NUM_ITERATIONS_SPEED; k++) { - subsample(1, 1)(luma_pels_ref, CFL_BUF_LINE, sub_luma_pels_ref, width, - height); + subsample(tx_size)(luma_pels_ref, CFL_BUF_LINE, sub_luma_pels_ref); } aom_usec_timer_mark(&timer); int elapsed_time = (int)aom_usec_timer_elapsed(&timer); @@ -431,7 +432,7 @@ #if HAVE_SSSE3 const subsample_param subsample_sizes_ssse3[] = { ALL_CFL_TX_SIZES( - get_subsample_lbd_fn_ssse3) }; + cfl_get_luma_subsampling_420_lbd_ssse3) }; const predict_param predict_sizes_ssse3[] = { ALL_CFL_TX_SIZES( get_predict_lbd_fn_ssse3) }; @@ -454,7 +455,7 @@ get_subtract_average_fn_avx2) }; const subsample_param subsample_sizes_avx2[] = { ALL_CFL_TX_SIZES( - get_subsample_lbd_fn_avx2) }; + cfl_get_luma_subsampling_420_lbd_avx2) }; const predict_param predict_sizes_avx2[] = { ALL_CFL_TX_SIZES( get_predict_lbd_fn_avx2) };