Fix logic to enable/disable length 4/8/16 inter DDT Use REPLACE_ADST4, REPLACE_ADST8, and REPLACE_ADST16 macros to turn DDT on or off for each length. These macros were not used since CONFIG_CORE_TX was introduced, which enabled length-4 DDT unintentionally. This patch has a little performance impact around 0.01% by disabling the length-4 DDTs. STATS_CHANGED
diff --git a/av1/common/idct.c b/av1/common/idct.c index 8eb5a02..ba435ea 100644 --- a/av1/common/idct.c +++ b/av1/common/idct.c
@@ -914,10 +914,16 @@ #if CONFIG_INTER_DDT if (txfm_param->use_ddt) { - if (tx_type_row == DST7 || tx_type_row == DCT8) { + const int use_ddt_row = (width == 4 && REPLACE_ADST4) || + (width == 8 && REPLACE_ADST8) || + (width == 16 && REPLACE_ADST16); + if (use_ddt_row && (tx_type_row == DST7 || tx_type_row == DCT8)) { tx_type_row = (tx_type_row == DST7) ? DDTX : FDDT; } - if (tx_type_col == DST7 || tx_type_col == DCT8) { + const int use_ddt_col = (height == 4 && REPLACE_ADST4) || + (height == 8 && REPLACE_ADST8) || + (height == 16 && REPLACE_ADST16); + if (use_ddt_col && (tx_type_col == DST7 || tx_type_col == DCT8)) { tx_type_col = (tx_type_col == DST7) ? DDTX : FDDT; } }
diff --git a/av1/common/txb_common.c b/av1/common/txb_common.c index 3c92b2d..08a323d 100644 --- a/av1/common/txb_common.c +++ b/av1/common/txb_common.c
@@ -2060,806 +2060,66 @@ #if CONFIG_INTER_DDT const int tx_kernel_ddtx_size4[TXFM_DIRECTIONS][4][4] = { - { - { - 2, - 20, - 72, - 104, - }, - { - 14, - 68, - 81, - -69, - }, - { - 67, - 86, - -61, - 25, - }, - { - 108, - -61, - 27, - -8, - }, - }, - { - { - 2, - 20, - 72, - 104, - }, - { - 14, - 68, - 81, - -69, - }, - { - 67, - 86, - -61, - 25, - }, - { - 108, - -61, - 27, - -8, - }, - }, + { { 2, 20, 72, 104 }, + { 14, 68, 81, -69 }, + { 67, 86, -61, 25 }, + { 108, -61, 27, -8 } }, + { { 2, 20, 72, 104 }, + { 14, 68, 81, -69 }, + { 67, 86, -61, 25 }, + { 108, -61, 27, -8 } }, }; const int tx_kernel_ddtx_size8[TXFM_DIRECTIONS][8][8] = { - { - { - 4, - 7, - 15, - 33, - 65, - 98, - 106, - 80, - }, - { - 6, - 14, - 36, - 77, - 100, - 45, - -57, - -98, - }, - { - 22, - 48, - 85, - 88, - 0, - -86, - -23, - 82, - }, - { - 57, - 94, - 76, - -26, - -73, - 34, - 54, - -66, - }, - { - 96, - 73, - -43, - -69, - 55, - 20, - -71, - 53, - }, - { - 103, - -17, - -80, - 56, - 15, - -66, - 75, - -41, - }, - { - 78, - -79, - 7, - 56, - -82, - 79, - -56, - 26, - }, - { - 56, - -96, - 98, - -77, - 54, - -33, - 19, - -6, - }, - }, - { - { - 4, - 7, - 15, - 33, - 65, - 98, - 106, - 80, - }, - { - 6, - 14, - 36, - 77, - 100, - 45, - -57, - -98, - }, - { - 22, - 48, - 85, - 88, - 0, - -86, - -23, - 82, - }, - { - 57, - 94, - 76, - -26, - -73, - 34, - 54, - -66, - }, - { - 96, - 73, - -43, - -69, - 55, - 20, - -71, - 53, - }, - { - 103, - -17, - -80, - 56, - 15, - -66, - 75, - -41, - }, - { - 78, - -79, - 7, - 56, - -82, - 79, - -56, - 26, - }, - { - 56, - -96, - 98, - -77, - 54, - -33, - 19, - -6, - }, - }, + { { 4, 7, 15, 33, 65, 98, 106, 80 }, + { 6, 14, 36, 77, 100, 45, -57, -98 }, + { 22, 48, 85, 88, 0, -86, -23, 82 }, + { 57, 94, 76, -26, -73, 34, 54, -66 }, + { 96, 73, -43, -69, 55, 20, -71, 53 }, + { 103, -17, -80, 56, 15, -66, 75, -41 }, + { 78, -79, 7, 56, -82, 79, -56, 26 }, + { 56, -96, 98, -77, 54, -33, 19, -6 } }, + { { 4, 7, 15, 33, 65, 98, 106, 80 }, + { 6, 14, 36, 77, 100, 45, -57, -98 }, + { 22, 48, 85, 88, 0, -86, -23, 82 }, + { 57, 94, 76, -26, -73, 34, 54, -66 }, + { 96, 73, -43, -69, 55, 20, -71, 53 }, + { 103, -17, -80, 56, 15, -66, 75, -41 }, + { 78, -79, 7, 56, -82, 79, -56, 26 }, + { 56, -96, 98, -77, 54, -33, 19, -6 } }, }; const int tx_kernel_ddtx_size16[TXFM_DIRECTIONS][16][16] = { - { - { - 12, - 15, - 19, - 23, - 30, - 39, - 51, - 66, - 78, - 88, - 94, - 97, - 93, - 83, - 68, - 50, - }, - { - 17, - 23, - 30, - 38, - 48, - 61, - 76, - 87, - 83, - 59, - 19, - -30, - -73, - -99, - -99, - -76, - }, - { - 37, - 49, - 60, - 69, - 75, - 75, - 61, - 29, - -18, - -67, - -96, - -83, - -28, - 40, - 84, - 83, - }, - { - 45, - 60, - 69, - 73, - 66, - 40, - -8, - -65, - -91, - -57, - 21, - 86, - 81, - 8, - -69, - -90, - }, - { - 47, - 60, - 61, - 49, - 19, - -29, - -77, - -83, - -16, - 75, - 93, - 3, - -92, - -74, - 32, - 97, - }, - { - 60, - 74, - 64, - 28, - -31, - -87, - -82, - 4, - 88, - 54, - -55, - -77, - 29, - 88, - 3, - -86, - }, - { - 64, - 70, - 40, - -19, - -79, - -78, - 11, - 92, - 28, - -85, - -41, - 82, - 39, - -83, - -37, - 83, - }, - { - 82, - 73, - 3, - -80, - -91, - 10, - 94, - 18, - -84, - -5, - 80, - -17, - -70, - 47, - 55, - -68, - }, - { - 89, - 48, - -53, - -96, - -5, - 89, - 16, - -83, - 12, - 75, - -51, - -43, - 81, - -14, - -75, - 67, - }, - { - 100, - 9, - -99, - -45, - 84, - 36, - -81, - 4, - 73, - -60, - -17, - 76, - -55, - -21, - 81, - -56, - }, - { - 92, - -35, - -91, - 42, - 71, - -69, - -22, - 85, - -60, - -17, - 77, - -70, - 11, - 56, - -83, - 49, - }, - { - 84, - -71, - -46, - 88, - -16, - -67, - 79, - -22, - -46, - 84, - -68, - 15, - 46, - -83, - 82, - -40, - }, - { - 69, - -83, - 2, - 75, - -78, - 18, - 50, - -85, - 81, - -43, - -6, - 53, - -81, - 88, - -69, - 32, - }, - { - 50, - -79, - 47, - 14, - -60, - 67, - -37, - -6, - 49, - -80, - 98, - -99, - 90, - -71, - 48, - -19, - }, - { - 51, - -89, - 73, - -17, - -45, - 89, - -103, - 97, - -83, - 71, - -56, - 44, - -31, - 22, - -11, - 5, - }, - { - 44, - -95, - 124, - -126, - 108, - -81, - 54, - -30, - 16, - -6, - 1, - 3, - -4, - 5, - -3, - 2, - }, - }, - { - { - 12, - 15, - 19, - 23, - 30, - 39, - 51, - 66, - 78, - 88, - 94, - 97, - 93, - 83, - 68, - 50, - }, - { - 17, - 23, - 30, - 38, - 48, - 61, - 76, - 87, - 83, - 59, - 19, - -30, - -73, - -99, - -99, - -76, - }, - { - 37, - 49, - 60, - 69, - 75, - 75, - 61, - 29, - -18, - -67, - -96, - -83, - -28, - 40, - 84, - 83, - }, - { - 45, - 60, - 69, - 73, - 66, - 40, - -8, - -65, - -91, - -57, - 21, - 86, - 81, - 8, - -69, - -90, - }, - { - 47, - 60, - 61, - 49, - 19, - -29, - -77, - -83, - -16, - 75, - 93, - 3, - -92, - -74, - 32, - 97, - }, - { - 60, - 74, - 64, - 28, - -31, - -87, - -82, - 4, - 88, - 54, - -55, - -77, - 29, - 88, - 3, - -86, - }, - { - 64, - 70, - 40, - -19, - -79, - -78, - 11, - 92, - 28, - -85, - -41, - 82, - 39, - -83, - -37, - 83, - }, - { - 82, - 73, - 3, - -80, - -91, - 10, - 94, - 18, - -84, - -5, - 80, - -17, - -70, - 47, - 55, - -68, - }, - { - 89, - 48, - -53, - -96, - -5, - 89, - 16, - -83, - 12, - 75, - -51, - -43, - 81, - -14, - -75, - 67, - }, - { - 100, - 9, - -99, - -45, - 84, - 36, - -81, - 4, - 73, - -60, - -17, - 76, - -55, - -21, - 81, - -56, - }, - { - 92, - -35, - -91, - 42, - 71, - -69, - -22, - 85, - -60, - -17, - 77, - -70, - 11, - 56, - -83, - 49, - }, - { - 84, - -71, - -46, - 88, - -16, - -67, - 79, - -22, - -46, - 84, - -68, - 15, - 46, - -83, - 82, - -40, - }, - { - 69, - -83, - 2, - 75, - -78, - 18, - 50, - -85, - 81, - -43, - -6, - 53, - -81, - 88, - -69, - 32, - }, - { - 50, - -79, - 47, - 14, - -60, - 67, - -37, - -6, - 49, - -80, - 98, - -99, - 90, - -71, - 48, - -19, - }, - { - 51, - -89, - 73, - -17, - -45, - 89, - -103, - 97, - -83, - 71, - -56, - 44, - -31, - 22, - -11, - 5, - }, - { - 44, - -95, - 124, - -126, - 108, - -81, - 54, - -30, - 16, - -6, - 1, - 3, - -4, - 5, - -3, - 2, - }, - }, + { { 12, 15, 19, 23, 30, 39, 51, 66, 78, 88, 94, 97, 93, 83, 68, 50 }, + { 17, 23, 30, 38, 48, 61, 76, 87, 83, 59, 19, -30, -73, -99, -99, -76 }, + { 37, 49, 60, 69, 75, 75, 61, 29, -18, -67, -96, -83, -28, 40, 84, 83 }, + { 45, 60, 69, 73, 66, 40, -8, -65, -91, -57, 21, 86, 81, 8, -69, -90 }, + { 47, 60, 61, 49, 19, -29, -77, -83, -16, 75, 93, 3, -92, -74, 32, 97 }, + { 60, 74, 64, 28, -31, -87, -82, 4, 88, 54, -55, -77, 29, 88, 3, -86 }, + { 64, 70, 40, -19, -79, -78, 11, 92, 28, -85, -41, 82, 39, -83, -37, 83 }, + { 82, 73, 3, -80, -91, 10, 94, 18, -84, -5, 80, -17, -70, 47, 55, -68 }, + { 89, 48, -53, -96, -5, 89, 16, -83, 12, 75, -51, -43, 81, -14, -75, 67 }, + { 100, 9, -99, -45, 84, 36, -81, 4, 73, -60, -17, 76, -55, -21, 81, -56 }, + { 92, -35, -91, 42, 71, -69, -22, 85, -60, -17, 77, -70, 11, 56, -83, 49 }, + { 84, -71, -46, 88, -16, -67, 79, -22, -46, 84, -68, 15, 46, -83, 82, -40 }, + { 69, -83, 2, 75, -78, 18, 50, -85, 81, -43, -6, 53, -81, 88, -69, 32 }, + { 50, -79, 47, 14, -60, 67, -37, -6, 49, -80, 98, -99, 90, -71, 48, -19 }, + { 51, -89, 73, -17, -45, 89, -103, 97, -83, 71, -56, 44, -31, 22, -11, 5 }, + { 44, -95, 124, -126, 108, -81, 54, -30, 16, -6, 1, 3, -4, 5, -3, 2 } }, + { { 12, 15, 19, 23, 30, 39, 51, 66, 78, 88, 94, 97, 93, 83, 68, 50 }, + { 17, 23, 30, 38, 48, 61, 76, 87, 83, 59, 19, -30, -73, -99, -99, -76 }, + { 37, 49, 60, 69, 75, 75, 61, 29, -18, -67, -96, -83, -28, 40, 84, 83 }, + { 45, 60, 69, 73, 66, 40, -8, -65, -91, -57, 21, 86, 81, 8, -69, -90 }, + { 47, 60, 61, 49, 19, -29, -77, -83, -16, 75, 93, 3, -92, -74, 32, 97 }, + { 60, 74, 64, 28, -31, -87, -82, 4, 88, 54, -55, -77, 29, 88, 3, -86 }, + { 64, 70, 40, -19, -79, -78, 11, 92, 28, -85, -41, 82, 39, -83, -37, 83 }, + { 82, 73, 3, -80, -91, 10, 94, 18, -84, -5, 80, -17, -70, 47, 55, -68 }, + { 89, 48, -53, -96, -5, 89, 16, -83, 12, 75, -51, -43, 81, -14, -75, 67 }, + { 100, 9, -99, -45, 84, 36, -81, 4, 73, -60, -17, 76, -55, -21, 81, -56 }, + { 92, -35, -91, 42, 71, -69, -22, 85, -60, -17, 77, -70, 11, 56, -83, 49 }, + { 84, -71, -46, 88, -16, -67, 79, -22, -46, 84, -68, 15, 46, -83, 82, -40 }, + { 69, -83, 2, 75, -78, 18, 50, -85, 81, -43, -6, 53, -81, 88, -69, 32 }, + { 50, -79, 47, 14, -60, 67, -37, -6, 49, -80, 98, -99, 90, -71, 48, -19 }, + { 51, -89, 73, -17, -45, 89, -103, 97, -83, 71, -56, 44, -31, 22, -11, 5 }, + { 44, -95, 124, -126, 108, -81, 54, -30, 16, -6, 1, 3, -4, 5, -3, 2 } }, }; #endif -#endif // CONFIG_CORE_TX \ No newline at end of file +#endif // CONFIG_CORE_TX
diff --git a/av1/common/x86/highbd_inv_txfm_avx2.c b/av1/common/x86/highbd_inv_txfm_avx2.c index e30ffc5..7ae9359 100644 --- a/av1/common/x86/highbd_inv_txfm_avx2.c +++ b/av1/common/x86/highbd_inv_txfm_avx2.c
@@ -9500,10 +9500,16 @@ #if CONFIG_INTER_DDT if (txfm_param->use_ddt) { - if (tx_type_row == DST7 || tx_type_row == DCT8) { + const int use_ddt_row = (width == 4 && REPLACE_ADST4) || + (width == 8 && REPLACE_ADST8) || + (width == 16 && REPLACE_ADST16); + if (use_ddt_row && (tx_type_row == DST7 || tx_type_row == DCT8)) { tx_type_row = (tx_type_row == DST7) ? DDTX : FDDT; } - if (tx_type_col == DST7 || tx_type_col == DCT8) { + const int use_ddt_col = (height == 4 && REPLACE_ADST4) || + (height == 8 && REPLACE_ADST8) || + (height == 16 && REPLACE_ADST16); + if (use_ddt_col && (tx_type_col == DST7 || tx_type_col == DCT8)) { tx_type_col = (tx_type_col == DST7) ? DDTX : FDDT; } }
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c index d04f88c..6e0ecc6 100644 --- a/av1/encoder/hybrid_fwd_txfm.c +++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -1295,10 +1295,16 @@ #if CONFIG_INTER_DDT if (txfm_param->use_ddt) { - if (tx_type_row == DST7 || tx_type_row == DCT8) { + const int use_ddt_row = (width == 4 && REPLACE_ADST4) || + (width == 8 && REPLACE_ADST8) || + (width == 16 && REPLACE_ADST16); + if (use_ddt_row && (tx_type_row == DST7 || tx_type_row == DCT8)) { tx_type_row = (tx_type_row == DST7) ? DDTX : FDDT; } - if (tx_type_col == DST7 || tx_type_col == DCT8) { + const int use_ddt_col = (height == 4 && REPLACE_ADST4) || + (height == 8 && REPLACE_ADST8) || + (height == 16 && REPLACE_ADST16); + if (use_ddt_col && (tx_type_col == DST7 || tx_type_col == DCT8)) { tx_type_col = (tx_type_col == DST7) ? DDTX : FDDT; } }
diff --git a/av1/encoder/x86/highbd_fwd_txfm_avx2.c b/av1/encoder/x86/highbd_fwd_txfm_avx2.c index 8bb80a2..e3bcf7e 100644 --- a/av1/encoder/x86/highbd_fwd_txfm_avx2.c +++ b/av1/encoder/x86/highbd_fwd_txfm_avx2.c
@@ -9288,10 +9288,16 @@ #if CONFIG_INTER_DDT if (txfm_param->use_ddt) { - if (tx_type_row == DST7 || tx_type_row == DCT8) { + const int use_ddt_row = (width == 4 && REPLACE_ADST4) || + (width == 8 && REPLACE_ADST8) || + (width == 16 && REPLACE_ADST16); + if (use_ddt_row && (tx_type_row == DST7 || tx_type_row == DCT8)) { tx_type_row = (tx_type_row == DST7) ? DDTX : FDDT; } - if (tx_type_col == DST7 || tx_type_col == DCT8) { + const int use_ddt_col = (height == 4 && REPLACE_ADST4) || + (height == 8 && REPLACE_ADST8) || + (height == 16 && REPLACE_ADST16); + if (use_ddt_col && (tx_type_col == DST7 || tx_type_col == DCT8)) { tx_type_col = (tx_type_col == DST7) ? DDTX : FDDT; } }