commit f1cf71785ec789fc2449333963ab380cc8ca89f0
author Luc Trudeau <luc@trud.ca> Tue Sep 19 13:32:05 2017 -0400
committer Luc Trudeau <luc@trud.ca> Tue Sep 19 21:31:47 2017 +0000
tree 9af8b1657f26575fcb4956b43f216c844e04a7ee
parent 03b7e7d553cdf40bf51410b715ab2bbfaf5bef0c

[CFL] Use MAX_NUM_TXB_SQUARE instead of MAX_NUM_TXB

MAX_NUM_TXB represents the maximum number of transform blocks in 1
dimension. CfL requires the maximum number of transform blocks in 2
dimensions. As such, we now use MAX_NUM_TXB_SQUARE instead of the
erroneous MAX_NUM_TXB for the average buffer.

There's an assert guarding for overrun on the average buffer.
This fix stops assertions on 4:4:4 sequences. This patch does
not alter AWCY results as AWCY only covers 4:2:0 sequences (for
which MAX_NUM_TXB turns out to be sufficient). 

Change-Id: I628db0131f60abc2d06cbbe3fe3dc40e28894ce3

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 23ea3b1..007e039 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -714,7 +714,7 @@
   //   * Max error will be 1/16th.
   // Note: 3 is chosen so that y_averages fits in 15 bits when 12 bit input is
   // used
-  int y_averages_q3[MAX_NUM_TXB];
+  int y_averages_q3[MAX_NUM_TXB_SQUARE];
   int y_averages_stride;
 
   int are_parameters_computed;