commit 6e4955d47fe98de9889fa19963665a3cb1e3c4cd
author Jingning Han <jingning@google.com> Tue May 30 22:54:48 2017 -0700
committer Jingning Han <jingning@google.com> Thu Jun 01 18:47:42 2017 +0000
tree ace8cf9c633d6916333ef1a44fe548d9edf4dfd8
parent eaddeee1ff0351d54b2659982639ea60fcfe02a9

Rework loop filter tx size selection

Update and capture the effective transform block size per color
plane.

Change-Id: Ib6e0e7abb3973db6b8d511ee7c9948aaab048788

av1/common/onyxc_int.h

diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index 795af30..38750c4 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -386,7 +386,8 @@
   ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
 #if CONFIG_VAR_TX
   TXFM_CONTEXT *above_txfm_context;
-  TXFM_CONTEXT left_txfm_context[2 * MAX_MIB_SIZE];
+  TXFM_CONTEXT *top_txfm_context[MAX_MB_PLANE];
+  TXFM_CONTEXT left_txfm_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
 #endif
   int above_context_alloc_cols;
 
@@ -928,6 +929,21 @@
   for (i = 0; i < bw; ++i) above_ctx[i] = txw;
 }
 
+static INLINE TX_SIZE get_sqr_tx_size(int tx_dim) {
+  TX_SIZE tx_size;
+  switch (tx_dim) {
+#if CONFIG_EXT_PARTITION
+    case 128:
+#endif
+    case 64:
+    case 32: tx_size = TX_32X32; break;
+    case 16: tx_size = TX_16X16; break;
+    case 8: tx_size = TX_8X8; break;
+    default: tx_size = TX_4X4;
+  }
+  return tx_size;
+}
+
 static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx,
                                          TXFM_CONTEXT *left_ctx,
                                          BLOCK_SIZE bsize, TX_SIZE tx_size) {
@@ -935,22 +951,13 @@
   const uint8_t txh = tx_size_high[tx_size];
   const int above = *above_ctx < txw;
   const int left = *left_ctx < txh;
-  TX_SIZE max_tx_size = max_txsize_lookup[bsize];
   int category = TXFM_PARTITION_CONTEXTS - 1;
 
   // dummy return, not used by others.
   if (tx_size <= TX_4X4) return 0;
 
-  switch (AOMMAX(block_size_wide[bsize], block_size_high[bsize])) {
-#if CONFIG_EXT_PARTITION
-    case 128:
-#endif
-    case 64:
-    case 32: max_tx_size = TX_32X32; break;
-    case 16: max_tx_size = TX_16X16; break;
-    case 8: max_tx_size = TX_8X8; break;
-    default: assert(max_tx_size == max_txsize_lookup[bsize]); assert(0);
-  }
+  TX_SIZE max_tx_size =
+      get_sqr_tx_size(AOMMAX(block_size_wide[bsize], block_size_high[bsize]));
 
   if (max_tx_size >= TX_8X8) {
     category = (tx_size != max_tx_size && max_tx_size > TX_8X8) +