[NORMATIVE] Fix txfm context for rect blocks

Make splittings at the first level and the second level falls into
different categories

BUG=aomedia:1160

Change-Id: If3066cb3a18a2c4f2ab4e7b8beb645afceca206f

av1/common/onyxc_int.h

diff --git a/av1/common/onyxc_int.h b/av1/common/onyxc_int.h
index afa31f0..5fb3e27 100644
--- a/av1/common/onyxc_int.h
+++ b/av1/common/onyxc_int.h
@@ -1319,8 +1319,9 @@
       get_sqr_tx_size(AOMMAX(block_size_wide[bsize], block_size_high[bsize]));
 
   if (max_tx_size >= TX_8X8) {
-    category = (tx_size != max_tx_size && max_tx_size > TX_8X8) +
-               (TX_SIZES - 1 - max_tx_size) * 2;
+    category =
+        (txsize_sqr_up_map[tx_size] != max_tx_size && max_tx_size > TX_8X8) +
+        (TX_SIZES - 1 - max_tx_size) * 2;
   }
   if (category == TXFM_PARTITION_CONTEXTS - 1) return category;
   return category * 3 + above + left;

test/decode_to_md5.sh

diff --git a/test/decode_to_md5.sh b/test/decode_to_md5.sh
index 3d15417..3df232b 100755
--- a/test/decode_to_md5.sh
+++ b/test/decode_to_md5.sh
@@ -57,7 +57,7 @@
 
 decode_to_md5_av1() {
   # expected MD5 sum for the last frame.
-  local expected_md5="887535ba1b01d6f7593ce7f4e9d86e12"
+  local expected_md5="be1cd883ce2cefa722be2dfec20318e1"
   local file="${AV1_IVF_FILE}"
 
   # TODO(urvang): Check in the encoded file (like libvpx does) to avoid