cb4x4: Fix indexing of tx_size counts.
Create named constants for the minimum TX_SIZE used in the luma
plane, as well as the minimum allowed TX_SIZE for which we have to
explicitly code a selection (i.e., one larger than the minimum luma
TX_SIZE).
Then, use these constants consistently with tx_size_to_depth() to
index the tx_size counts in FRAME_COUNTS, and also consistently use
TX_SIZE named constants to index the tx_size_implied counts.
Failing to index these counts correctly can, e.g., cause tx_mode to
be chosen incorrectly.
Change-Id: I706a62a33e2282e67c97a68bade87fb8023ec13b
diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 826cc51..76e0cec 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1096,11 +1096,11 @@
void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
static INLINE int tx_size_to_depth(TX_SIZE tx_size) {
- return (int)(tx_size - TX_4X4);
+ return (int)(tx_size - TX_SIZE_LUMA_MIN);
}
static INLINE TX_SIZE depth_to_tx_size(int depth) {
- return (TX_SIZE)(depth + TX_4X4);
+ return (TX_SIZE)(depth + TX_SIZE_LUMA_MIN);
}
static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,
diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h
index 2fb90fb..f65db70 100644
--- a/av1/common/entropymode.h
+++ b/av1/common/entropymode.h
@@ -401,7 +401,7 @@
// to use forward updates for the coeff probs, and as such it does not really
// belong into this structure.
unsigned int tx_size_totals[TX_SIZES];
- unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][TX_SIZES];
+ unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
unsigned int quarter_tx_size[2];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_RECT_TX_EXT
diff --git a/av1/common/enums.h b/av1/common/enums.h
index 4f53384..d59b84f 100644
--- a/av1/common/enums.h
+++ b/av1/common/enums.h
@@ -175,7 +175,12 @@
TX_INVALID = 255 // Invalid transform size
} TX_SIZE;
-#define MAX_TX_DEPTH (TX_SIZES - 1 - TX_4X4)
+#define TX_SIZE_LUMA_MIN (TX_4X4)
+/* We don't need to code a transform size unless the allowed size is at least
+ one more than the minimum. */
+#define TX_SIZE_CTX_MIN (TX_SIZE_LUMA_MIN + 1)
+
+#define MAX_TX_DEPTH (TX_SIZES - TX_SIZE_CTX_MIN)
#define MAX_TX_SIZE_LOG2 (5 + CONFIG_TX64X64)
#define MAX_TX_SIZE (1 << MAX_TX_SIZE_LOG2)
diff --git a/av1/common/pred_common.h b/av1/common/pred_common.h
index 893813e..e171703 100644
--- a/av1/common/pred_common.h
+++ b/av1/common/pred_common.h
@@ -231,7 +231,9 @@
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
if (tx_size == plane_tx_size) {
- ++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][tx_size];
+ int depth;
+ depth = tx_size_to_depth(tx_size);
+ ++xd->counts->tx_size[max_tx_size - TX_SIZE_CTX_MIN][ctx][depth];
mbmi->tx_size = tx_size;
} else {
int bsl = b_width_log2_lookup[bsize];
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 0f4af70..549c230 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5528,41 +5528,50 @@
int count32x32_32x32p = 0, count32x32_lp = 0;
int count64x64_64x64p = 0;
for (i = 0; i < TX_SIZE_CONTEXTS; ++i) {
+ int depth;
// counts->tx_size[max_depth][context_idx][this_depth_level]
- count4x4 += counts->tx_size[0][i][0];
- count4x4 += counts->tx_size[1][i][0];
- count4x4 += counts->tx_size[2][i][0];
- count4x4 += counts->tx_size[3][i][0];
+ depth = tx_size_to_depth(TX_4X4);
+ count4x4 += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
+ count4x4 += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
+ count4x4 += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
+ count4x4 += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
- count8x8_8x8p += counts->tx_size[0][i][1];
- count8x8_lp += counts->tx_size[1][i][1];
- count8x8_lp += counts->tx_size[2][i][1];
- count8x8_lp += counts->tx_size[3][i][1];
+ depth = tx_size_to_depth(TX_8X8);
+ count8x8_8x8p += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
+ count8x8_lp += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
+ count8x8_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
+ count8x8_lp += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
- count16x16_16x16p += counts->tx_size[1][i][2];
- count16x16_lp += counts->tx_size[2][i][2];
- count16x16_lp += counts->tx_size[3][i][2];
+ depth = tx_size_to_depth(TX_16X16);
+ count16x16_16x16p +=
+ counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
+ count16x16_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
+ count16x16_lp += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
- count32x32_32x32p += counts->tx_size[2][i][3];
- count32x32_lp += counts->tx_size[3][i][3];
+ depth = tx_size_to_depth(TX_32X32);
+ count32x32_32x32p +=
+ counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
+ count32x32_lp += counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
- count64x64_64x64p += counts->tx_size[3][i][4];
+ depth = tx_size_to_depth(TX_64X64);
+ count64x64_64x64p +=
+ counts->tx_size[TX_64X64 - TX_SIZE_CTX_MIN][i][depth];
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- count4x4 += counts->tx_size_implied[0][TX_4X4];
- count4x4 += counts->tx_size_implied[1][TX_4X4];
- count4x4 += counts->tx_size_implied[2][TX_4X4];
- count4x4 += counts->tx_size_implied[3][TX_4X4];
- count8x8_8x8p += counts->tx_size_implied[1][TX_8X8];
- count8x8_lp += counts->tx_size_implied[2][TX_8X8];
- count8x8_lp += counts->tx_size_implied[3][TX_8X8];
- count8x8_lp += counts->tx_size_implied[4][TX_8X8];
- count16x16_16x16p += counts->tx_size_implied[2][TX_16X16];
- count16x16_lp += counts->tx_size_implied[3][TX_16X16];
- count16x16_lp += counts->tx_size_implied[4][TX_16X16];
- count32x32_32x32p += counts->tx_size_implied[3][TX_32X32];
- count32x32_lp += counts->tx_size_implied[4][TX_32X32];
- count64x64_64x64p += counts->tx_size_implied[4][TX_64X64];
+ count4x4 += counts->tx_size_implied[TX_4X4][TX_4X4];
+ count4x4 += counts->tx_size_implied[TX_8X8][TX_4X4];
+ count4x4 += counts->tx_size_implied[TX_16X16][TX_4X4];
+ count4x4 += counts->tx_size_implied[TX_32X32][TX_4X4];
+ count8x8_8x8p += counts->tx_size_implied[TX_8X8][TX_8X8];
+ count8x8_lp += counts->tx_size_implied[TX_16X16][TX_8X8];
+ count8x8_lp += counts->tx_size_implied[TX_32X32][TX_8X8];
+ count8x8_lp += counts->tx_size_implied[TX_64X64][TX_8X8];
+ count16x16_16x16p += counts->tx_size_implied[TX_16X16][TX_16X16];
+ count16x16_lp += counts->tx_size_implied[TX_32X32][TX_16X16];
+ count16x16_lp += counts->tx_size_implied[TX_64X64][TX_16X16];
+ count32x32_32x32p += counts->tx_size_implied[TX_32X32][TX_32X32];
+ count32x32_lp += counts->tx_size_implied[TX_64X64][TX_32X32];
+ count64x64_64x64p += counts->tx_size_implied[TX_64X64][TX_64X64];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 &&
count32x32_lp == 0 && count32x32_32x32p == 0 &&
@@ -5614,30 +5623,37 @@
int count16x16_16x16p = 0, count16x16_lp = 0;
int count32x32 = 0;
for (i = 0; i < TX_SIZE_CONTEXTS; ++i) {
+ int depth;
// counts->tx_size[max_depth][context_idx][this_depth_level]
- count4x4 += counts->tx_size[0][i][0];
- count4x4 += counts->tx_size[1][i][0];
- count4x4 += counts->tx_size[2][i][0];
+ depth = tx_size_to_depth(TX_4X4);
+ count4x4 += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
+ count4x4 += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
+ count4x4 += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
- count8x8_8x8p += counts->tx_size[0][i][1];
- count8x8_lp += counts->tx_size[1][i][1];
- count8x8_lp += counts->tx_size[2][i][1];
+ depth = tx_size_to_depth(TX_8X8);
+ count8x8_8x8p += counts->tx_size[TX_8X8 - TX_SIZE_CTX_MIN][i][depth];
+ count8x8_lp += counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
+ count8x8_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
- count16x16_16x16p += counts->tx_size[1][i][2];
- count16x16_lp += counts->tx_size[2][i][2];
- count32x32 += counts->tx_size[2][i][3];
+ depth = tx_size_to_depth(TX_16X16);
+ count16x16_16x16p +=
+ counts->tx_size[TX_16X16 - TX_SIZE_CTX_MIN][i][depth];
+ count16x16_lp += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
+
+ depth = tx_size_to_depth(TX_32X32);
+ count32x32 += counts->tx_size[TX_32X32 - TX_SIZE_CTX_MIN][i][depth];
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
- count4x4 += counts->tx_size_implied[0][TX_4X4];
- count4x4 += counts->tx_size_implied[1][TX_4X4];
- count4x4 += counts->tx_size_implied[2][TX_4X4];
- count4x4 += counts->tx_size_implied[3][TX_4X4];
- count8x8_8x8p += counts->tx_size_implied[1][TX_8X8];
- count8x8_lp += counts->tx_size_implied[2][TX_8X8];
- count8x8_lp += counts->tx_size_implied[3][TX_8X8];
- count16x16_lp += counts->tx_size_implied[3][TX_16X16];
- count16x16_16x16p += counts->tx_size_implied[2][TX_16X16];
- count32x32 += counts->tx_size_implied[3][TX_32X32];
+ count4x4 += counts->tx_size_implied[TX_4X4][TX_4X4];
+ count4x4 += counts->tx_size_implied[TX_8X8][TX_4X4];
+ count4x4 += counts->tx_size_implied[TX_16X16][TX_4X4];
+ count4x4 += counts->tx_size_implied[TX_32X32][TX_4X4];
+ count8x8_8x8p += counts->tx_size_implied[TX_8X8][TX_8X8];
+ count8x8_lp += counts->tx_size_implied[TX_16X16][TX_8X8];
+ count8x8_lp += counts->tx_size_implied[TX_32X32][TX_8X8];
+ count16x16_16x16p += counts->tx_size_implied[TX_16X16][TX_16X16];
+ count16x16_lp += counts->tx_size_implied[TX_32X32][TX_16X16];
+ count32x32 += counts->tx_size_implied[TX_32X32][TX_32X32];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 &&
#if CONFIG_SUPERTX
