commit fe0fb1d9deccdfb715878e09999fc11ae91a396c
author Timothy B. Terriberry <tterriberry@mozilla.com> Thu May 18 12:15:16 2017 -0700
committer Tim Terriberry <tterriberry@mozilla.com> Sun May 21 22:42:30 2017 +0000
tree f5ff32087b1c6402395b871695208b627c7726c3
parent 2af277a442e40cf5c968df298b83525a37dde65f

cb4x4: Fix indexing of tx_size counts.

Create named constants for the minimum TX_SIZE used in the luma
plane, as well as the minimum allowed TX_SIZE for which we have to
explicitly code a selection (i.e., one larger than the minimum luma
TX_SIZE).

Then, use these constants consistently with tx_size_to_depth() to
index the tx_size counts in FRAME_COUNTS, and also consistently use
TX_SIZE named constants to index the tx_size_implied counts.

Failing to index these counts correctly can, e.g., cause tx_mode to
be chosen incorrectly.

Change-Id: I706a62a33e2282e67c97a68bade87fb8023ec13b

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 826cc51..76e0cec 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1096,11 +1096,11 @@
 void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
 
 static INLINE int tx_size_to_depth(TX_SIZE tx_size) {
-  return (int)(tx_size - TX_4X4);
+  return (int)(tx_size - TX_SIZE_LUMA_MIN);
 }
 
 static INLINE TX_SIZE depth_to_tx_size(int depth) {
-  return (TX_SIZE)(depth + TX_4X4);
+  return (TX_SIZE)(depth + TX_SIZE_LUMA_MIN);
 }
 
 static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,

av1/common/entropymode.h

diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 2fb90fb..f65db70 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -401,7 +401,7 @@
   // to use forward updates for the coeff probs, and as such it does not really
   // belong into this structure.
   unsigned int tx_size_totals[TX_SIZES];
-  unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][TX_SIZES];
+  unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
 #if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
   unsigned int quarter_tx_size[2];
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT

av1/common/enums.h

diff --git a/av1/common/enums.h b/av1/common/enums.h
index 4f53384..d59b84f 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -175,7 +175,12 @@
   TX_INVALID = 255    // Invalid transform size
 } TX_SIZE;
 
-#define MAX_TX_DEPTH (TX_SIZES - 1 - TX_4X4)
+#define TX_SIZE_LUMA_MIN (TX_4X4)
+/* We don't need to code a transform size unless the allowed size is at least
+   one more than the minimum. */
+#define TX_SIZE_CTX_MIN (TX_SIZE_LUMA_MIN + 1)
+
+#define MAX_TX_DEPTH (TX_SIZES - TX_SIZE_CTX_MIN)
 
 #define MAX_TX_SIZE_LOG2 (5 + CONFIG_TX64X64)
 #define MAX_TX_SIZE (1 << MAX_TX_SIZE_LOG2)

av1/common/pred_common.h

diff --git a/av1/common/pred_common.h b/av1/common/pred_common.h
index 893813e..e171703 100644
--- a/av1/common/pred_common.h
+++ b/av1/common/pred_common.h
@@ -231,7 +231,9 @@
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
   if (tx_size == plane_tx_size) {
-    ++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][tx_size];
+    int depth;
+    depth = tx_size_to_depth(tx_size);
+    ++xd->counts->tx_size[max_tx_size - TX_SIZE_CTX_MIN][ctx][depth];
     mbmi->tx_size = tx_size;
   } else {
     int bsl = b_width_log2_lookup[bsize];