commit 7167d952137f7b3c76a2c0bfdd64016950f27aa8
author Hui Su <huisu@google.com> Thu Feb 01 16:33:12 2018 -0800
committer Hui Su <huisu@google.com> Fri Feb 02 18:19:31 2018 +0000
tree 242dd3b5f0d3ad321d266bcde75f1b898f0e1f98
parent e5d166efb50df622545512a95a933ef984b470f3

Reduce memory usage of inter_tx_size[] in MB_MODE_INFO

Reduce the length of inter_tx_size[] from 1024 to 16.

On a cif test sequence,
encoder memory consumption decreases by 18% (380MB -> 312MB);
decoder memory consumption decreases by 56% (21.4MB -> 9.4MB).

Change-Id: I42928eb9312748f96f4393c8d8040791f38f98b6

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index e5c4d6a..ff1d8dc 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -177,8 +177,6 @@
                                 aom_writer *w) {
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
   (void)cm;
-  const int tx_row = blk_row >> 1;
-  const int tx_col = blk_col >> 1;
   const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
   const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);
 
@@ -190,12 +188,13 @@
     return;
   }
 
-  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
-                                   xd->left_txfm_context + blk_row,
-                                   mbmi->sb_type, tx_size);
-
+  const int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
+                                         xd->left_txfm_context + blk_row,
+                                         mbmi->sb_type, tx_size);
+  const int txb_size_index =
+      av1_get_txb_size_index(mbmi->sb_type, blk_row, blk_col);
   const int write_txfm_partition =
-      tx_size == mbmi->inter_tx_size[tx_row][tx_col];
+      tx_size == mbmi->inter_tx_size[txb_size_index];
   if (write_txfm_partition) {
     aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
 
@@ -464,8 +463,6 @@
                             int block, int blk_row, int blk_col,
                             TX_SIZE tx_size, TOKEN_STATS *token_stats) {
   const struct macroblockd_plane *const pd = &xd->plane[plane];
-  const int tx_row = blk_row >> (1 - pd->subsampling_y);
-  const int tx_col = blk_col >> (1 - pd->subsampling_x);
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
@@ -473,7 +470,8 @@
 
   const TX_SIZE plane_tx_size =
       plane ? av1_get_uv_tx_size(mbmi, pd->subsampling_x, pd->subsampling_y)
-            : mbmi->inter_tx_size[tx_row][tx_col];
+            : mbmi->inter_tx_size[av1_get_txb_size_index(plane_bsize, blk_row,
+                                                         blk_col)];
 
   if (tx_size == plane_tx_size || plane) {
     TOKEN_STATS tmp_token_stats;