commit 1fa24467aa88d96e3f32985552d4757cd042d7a4
author Debargha Mukherjee <debargha@google.com> Thu Oct 19 14:33:39 2017 -0700
committer Debargha Mukherjee <debargha@google.com> Fri Oct 20 02:16:42 2017 +0000
tree 88ed24117cb5fe4f05a7d8ca460301f025a08b3b
parent 9511a433b2b5c9f8bb3ba2c778a949a0a0385d79

Fix assert failure with tx64x64 and ext-partiton

Fixes an assert failure when 32x64 and 64x32 partitions are used
with tx64x64 when ext-partition is also on.

BUG=aomedia:935

Change-Id: I3bf50aeab58ec4ba2c9892e6eb18cf60e425fa42

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index fb7da68..b50e5e1 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -1446,7 +1446,9 @@
   // a subsampled BLOCK_128X128 then the lookup above will give TX_64X64. That
   // mustn't be used for the subsampled plane (because it would be bigger than
   // a 64x64 luma block) so we round down to TX_32X32.
-  if (subsampled && max_txsize == TX_64X64) max_txsize = TX_32X32;
+  if (subsampled && txsize_sqr_up_map[max_txsize] == TX_64X64) {
+    max_txsize = TX_32X32;
+  }
 #else
   (void)subsampled;
 #endif