Change tx_size encoding for intra modes

Conveys depth from the largest transform size instead of the
actual transform size. Besides, the max depth is now limited
by the macro MAX_TX_DPETH set at 2.

Results: BDRATE lowres (30 frames): -0.005%

Change-Id: I1ccbac8ee18c77b816a6a8f500abfaa7892b21de

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 9f7e314..23b60c5 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -322,18 +322,22 @@
   (void)cm;
   if (block_signals_txsize(bsize)) {
     const TX_SIZE tx_size = mbmi->tx_size;
-    const int is_inter = is_inter_block(mbmi);
     const int tx_size_ctx = get_tx_size_context(xd);
-    const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
-                                         : intra_tx_size_cat_lookup[bsize];
+    const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
-    const int depth = tx_size_to_depth(coded_tx_size);
+    const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
+    const int max_depths = tx_size_cat_to_max_depth(tx_size_cat);
+
+    assert(coded_tx_size <= tx_size_cat + 1);
+    assert(depth >= 0 && depth <= max_depths);
+
+    assert(!is_inter_block(mbmi));
     assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
 
     aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
-                     tx_size_cat + 2);
+                     max_depths + 1);
 #if CONFIG_RECT_TX_EXT
-    if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
+    if (is_quarter_tx_allowed(xd, mbmi, 0) && tx_size != coded_tx_size)
 #if CONFIG_NEW_MULTISYMBOL
       aom_write_symbol(w, tx_size == quarter_txsize_lookup[bsize],
                        cm->fc->quarter_tx_size_cdf, 2);

av1/encoder/rd.c

diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index a44bc4a..6191136 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -219,7 +219,7 @@
   }
 #endif  // CONFIG_CFL
 
-  for (i = 0; i < MAX_TX_DEPTH; ++i)
+  for (i = 0; i < MAX_TX_CATS; ++i)
     for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
       av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
                                NULL);

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index d3090c4..08f052d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2357,7 +2357,7 @@
     const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                          : intra_tx_size_cat_lookup[bsize];
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
-    const int depth = tx_size_to_depth(coded_tx_size);
+    const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
     const int tx_size_ctx = get_tx_size_context(xd);
     int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
 #if CONFIG_RECT_TX_EXT
@@ -2739,7 +2739,7 @@
 
   if (tx_select) {
     start_tx = max_tx_size;
-    end_tx = (max_tx_size >= TX_32X32) ? TX_8X8 : TX_4X4;
+    end_tx = AOMMAX((int)TX_4X4, start_tx - MAX_TX_DEPTH + evaluate_rect_tx);
   } else {
     const TX_SIZE chosen_tx_size =
         tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);