commit 48f84dbd1c57ac475d4d7c13cc6b7cbd25290455
author Yushin Cho <ycho@mozilla.com> Mon Nov 07 21:20:17 2016 -0800
committer Yaowu Xu <yaowu@google.com> Tue Nov 08 08:15:57 2016 -0800
tree f1c357a43ae5e8c2baeaf7e20aa9000b3ff94cc0
parent f0481a590f3c90390fed8048a469d5fad1addc73

Fix the bug that PVQ commit broke dering

Since PVQ's max block size equals to the max transform size,
daala's definition of OD_BSIZE_MAX was changed from 5 down to 4 to
use AV1's max trasform size 32x32. However, dering also uses
OD_BSIZE_MAX and assumes its value is 5, which caused dering
not working.

Change-Id: I9d82bb24adc7d57552a8e0a8a7e798e77d96fd4b

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 91040db..dfde235 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -1391,13 +1391,13 @@
 #if PVQ_CHROMA_RD
   double save_pvq_lambda;
 #endif
-  DECLARE_ALIGNED(16, int16_t, coeff_pvq[OD_BSIZE_MAX * OD_BSIZE_MAX]);
-  DECLARE_ALIGNED(16, int16_t, ref_coeff_pvq[OD_BSIZE_MAX * OD_BSIZE_MAX]);
-  DECLARE_ALIGNED(16, int16_t, dqcoeff_pvq[OD_BSIZE_MAX * OD_BSIZE_MAX]);
+  DECLARE_ALIGNED(16, int16_t, coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
+  DECLARE_ALIGNED(16, int16_t, ref_coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
+  DECLARE_ALIGNED(16, int16_t, dqcoeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
 
-  DECLARE_ALIGNED(16, int32_t, in_int32[OD_BSIZE_MAX * OD_BSIZE_MAX]);
-  DECLARE_ALIGNED(16, int32_t, ref_int32[OD_BSIZE_MAX * OD_BSIZE_MAX]);
-  DECLARE_ALIGNED(16, int32_t, out_int32[OD_BSIZE_MAX * OD_BSIZE_MAX]);
+  DECLARE_ALIGNED(16, int32_t, in_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
+  DECLARE_ALIGNED(16, int32_t, ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
+  DECLARE_ALIGNED(16, int32_t, out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
 
   *eob = 0;