Move all of LBD Daala TX to up-4, down-1 shift Now that tran_low_t is assumed to be 32 bit when Daala TX is active, there's no reason for multi-stage shifting to fit coefficients into 16 bits for the inter-tranform transpose matrix. Go to a consistent up by four, down by one shifting scheme for all TX block sizes. (Note this is for the current AV1 coefficient scaling scheme with av1_get_tx_scale and deeper coefficients for higher bitdepth input. Daala TX is moving to the long-intended constant-coefficient-depth in upcoming patches). subset 1: monty-4-1-baseline-s1@2017-11-11T05:57:15.857Z -> monty-4-1-test-s1@2017-11-11T05:57:52.983Z PSNR | PSNR Cb | PSNR Cr | PSNR HVS | SSIM | MS SSIM | CIEDE 2000 -0.0117 | -0.0246 | 0.0530 | 0.0238 | 0.0254 | 0.0447 | -0.0442 Change-Id: I2214e94ac822542c504d472276723277ed350abf
diff --git a/av1/common/idct.c b/av1/common/idct.c index dc07155..c3bde4c 100644 --- a/av1/common/idct.c +++ b/av1/common/idct.c
@@ -1008,9 +1008,9 @@ for (int i = 0; i < n2; ++i) { #if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 tran_low_t temp_in[16]; - for (int j = 0; j < n; j++) temp_in[j] = input[j] * 2; + for (int j = 0; j < n; j++) temp_in[j] = input[j] * 4; IHT_16x32[tx_type].rows(temp_in, outtmp); - for (int j = 0; j < n; ++j) tmp[j][i] = outtmp[j] * 4; + for (int j = 0; j < n; ++j) tmp[j][i] = outtmp[j]; #else IHT_16x32[tx_type].rows(input, outtmp); for (int j = 0; j < n; ++j) @@ -1030,7 +1030,7 @@ int d = i * stride + j; int s = j * outstride + i; #if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 - dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 4)); #else dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); #endif @@ -1095,9 +1095,9 @@ for (int i = 0; i < n; ++i) { #if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 tran_low_t temp_in[32]; - for (int j = 0; j < n2; j++) temp_in[j] = input[j] * 2; + for (int j = 0; j < n2; j++) temp_in[j] = input[j] * 4; IHT_32x16[tx_type].rows(temp_in, outtmp); - for (int j = 0; j < n2; ++j) tmp[j][i] = outtmp[j] * 4; + for (int j = 0; j < n2; ++j) tmp[j][i] = outtmp[j]; #else IHT_32x16[tx_type].rows(input, outtmp); for (int j = 0; j < n2; ++j) @@ -1117,7 +1117,7 @@ int d = i * stride + j; int s = j * outstride + i; #if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 - dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 4)); #else dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); #endif @@ -1360,7 +1360,7 @@ for (int i = 0; i < 32; ++i) { #if CONFIG_DAALA_TX32 tran_low_t temp_in[32]; - for (int j = 0; j < 32; j++) temp_in[j] = input[j] * 2; + for (int j = 0; j < 32; j++) temp_in[j] = input[j] * 4; IHT_32[tx_type].rows(temp_in, out[i]); #else IHT_32[tx_type].rows(input, out[i]); @@ -1369,15 +1369,8 @@ } // transpose - for (int i = 0; i < 32; i++) { - for (int j = 0; j < 32; j++) { -#if CONFIG_DAALA_TX32 - tmp[j][i] = out[i][j] * 4; -#else - tmp[j][i] = out[i][j]; -#endif - } - } + for (int i = 0; i < 32; i++) + for (int j = 0; j < 32; j++) tmp[j][i] = out[i][j]; // inverse transform column vectors for (int i = 0; i < 32; ++i) IHT_32[tx_type].cols(tmp[i], out[i]); @@ -1390,7 +1383,7 @@ int d = i * stride + j; int s = j * outstride + i; #if CONFIG_DAALA_TX32 - dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 4)); #else dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 6)); #endif @@ -1455,9 +1448,8 @@ for (int i = 0; i < 64; ++i) { #if CONFIG_DAALA_TX64 tran_low_t temp_in[64]; - for (int j = 0; j < 64; j++) temp_in[j] = input[j] * 2; + for (int j = 0; j < 64; j++) temp_in[j] = input[j] * 8; IHT_64[tx_type].rows(temp_in, out[i]); -// Do not rescale intermediate for Daala #else IHT_64[tx_type].rows(input, out[i]); for (int j = 0; j < 64; ++j) out[i][j] = ROUND_POWER_OF_TWO(out[i][j], 1); @@ -1483,7 +1475,7 @@ int d = i * stride + j; int s = j * outstride + i; #if CONFIG_DAALA_TX64 - dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 2)); + dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 4)); #else dest[d] = clip_pixel_add(dest[d], ROUND_POWER_OF_TWO(outp[s], 5)); #endif
diff --git a/av1/encoder/dct.c b/av1/encoder/dct.c index 6ae3080..43d059a 100644 --- a/av1/encoder/dct.c +++ b/av1/encoder/dct.c
@@ -1965,7 +1965,7 @@ ht.rows(temp_in, temp_out); for (j = 0; j < n; ++j) { #if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 - out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); + out[j * n2 + i] = temp_out[j]; #else out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 4); #endif @@ -1976,7 +1976,12 @@ for (i = 0; i < n; ++i) { for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; ht.cols(temp_in, temp_out); +#if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 + for (j = 0; j < n2; ++j) + output[i + j * n] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); +#else for (j = 0; j < n2; ++j) output[i + j * n] = temp_out[j]; +#endif } // Note: overall scale factor of transform is 4 times unitary } @@ -2049,7 +2054,7 @@ ht.cols(temp_in, temp_out); for (j = 0; j < n; ++j) { #if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 - out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); + out[j * n2 + i] = temp_out[j]; #else out[j * n2 + i] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 4); #endif @@ -2060,7 +2065,12 @@ for (i = 0; i < n; ++i) { for (j = 0; j < n2; ++j) temp_in[j] = out[j + i * n2]; ht.rows(temp_in, temp_out); +#if CONFIG_DAALA_TX16 && CONFIG_DAALA_TX32 + for (j = 0; j < n2; ++j) + output[j + i * n2] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 2); +#else for (j = 0; j < n2; ++j) output[j + i * n2] = temp_out[j]; +#endif } // Note: overall scale factor of transform is 4 times unitary } @@ -2489,8 +2499,7 @@ #if CONFIG_DAALA_TX64 for (j = 0; j < 64; ++j) temp_in[j] = input[j * stride + i] * 16; ht.cols(temp_in, temp_out); - for (j = 0; j < 64; ++j) - out[j * 64 + i] = (temp_out[j] + 1 + (temp_out[j] > 0)) >> 3; + for (j = 0; j < 64; ++j) out[j * 64 + i] = temp_out[j]; #else for (j = 0; j < 64; ++j) temp_in[j] = input[j * stride + i]; @@ -2506,7 +2515,7 @@ ht.rows(temp_in, temp_out); for (j = 0; j < 64; ++j) #if CONFIG_DAALA_TX64 - output[j + i * 64] = temp_out[j]; + output[j + i * 64] = ROUND_POWER_OF_TWO_SIGNED(temp_out[j], 3); #else output[j + i * 64] = (tran_low_t)((temp_out[j] + 1 + (temp_out[j] < 0)) >> 2);